net/core/dev.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *      NET3    Protocol independent device support routines.
   4  *
   5  *      Derived from the non IP parts of dev.c 1.0.19
   6  *              Authors:        Ross Biro
   7  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
   8  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
   9  *
  10  *      Additional Authors:
  11  *              Florian la Roche <rzsfl@rz.uni-sb.de>
  12  *              Alan Cox <gw4pts@gw4pts.ampr.org>
  13  *              David Hinds <dahinds@users.sourceforge.net>
  14  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
  15  *              Adam Sulmicki <adam@cfar.umd.edu>
  16  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
  17  *
  18  *      Changes:
  19  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
  20  *                                      to 2 if register_netdev gets called
  21  *                                      before net_dev_init & also removed a
  22  *                                      few lines of code in the process.
  23  *              Alan Cox        :       device private ioctl copies fields back.
  24  *              Alan Cox        :       Transmit queue code does relevant
  25  *                                      stunts to keep the queue safe.
  26  *              Alan Cox        :       Fixed double lock.
  27  *              Alan Cox        :       Fixed promisc NULL pointer trap
  28  *              ????????        :       Support the full private ioctl range
  29  *              Alan Cox        :       Moved ioctl permission check into
  30  *                                      drivers
  31  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
  32  *              Alan Cox        :       100 backlog just doesn't cut it when
  33  *                                      you start doing multicast video 8)
  34  *              Alan Cox        :       Rewrote net_bh and list manager.
  35  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
  36  *              Alan Cox        :       Took out transmit every packet pass
  37  *                                      Saved a few bytes in the ioctl handler
  38  *              Alan Cox        :       Network driver sets packet type before
  39  *                                      calling netif_rx. Saves a function
  40  *                                      call a packet.
  41  *              Alan Cox        :       Hashed net_bh()
  42  *              Richard Kooijman:       Timestamp fixes.
  43  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
  44  *              Alan Cox        :       Device lock protection.
  45  *              Alan Cox        :       Fixed nasty side effect of device close
  46  *                                      changes.
  47  *              Rudi Cilibrasi  :       Pass the right thing to
  48  *                                      set_mac_address()
  49  *              Dave Miller     :       32bit quantity for the device lock to
  50  *                                      make it work out on a Sparc.
  51  *              Bjorn Ekwall    :       Added KERNELD hack.
  52  *              Alan Cox        :       Cleaned up the backlog initialise.
  53  *              Craig Metz      :       SIOCGIFCONF fix if space for under
  54  *                                      1 device.
  55  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
  56  *                                      is no device open function.
  57  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
  58  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
  59  *              Cyrus Durgin    :       Cleaned for KMOD
  60  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
  61  *                                      A network device unload needs to purge
  62  *                                      the backlog queue.
  63  *      Paul Rusty Russell      :       SIOCSIFNAME
  64  *              Pekka Riikonen  :       Netdev boot-time settings code
  65  *              Andrew Morton   :       Make unregister_netdevice wait
  66  *                                      indefinitely on dev->refcnt
  67  *              J Hadi Salim    :       - Backlog queue sampling
  68  *                                      - netif_rx() feedback
  69  */
  70
  71 #include <linux/uaccess.h>
  72 #include <linux/bitops.h>
  73 #include <linux/capability.h>
  74 #include <linux/cpu.h>
  75 #include <linux/types.h>
  76 #include <linux/kernel.h>
  77 #include <linux/hash.h>
  78 #include <linux/slab.h>
  79 #include <linux/sched.h>
  80 #include <linux/sched/mm.h>
  81 #include <linux/mutex.h>
  82 #include <linux/rwsem.h>
  83 #include <linux/string.h>
  84 #include <linux/mm.h>
  85 #include <linux/socket.h>
  86 #include <linux/sockios.h>
  87 #include <linux/errno.h>
  88 #include <linux/interrupt.h>
  89 #include <linux/if_ether.h>
  90 #include <linux/netdevice.h>
  91 #include <linux/etherdevice.h>
  92 #include <linux/ethtool.h>
  93 #include <linux/skbuff.h>
  94 #include <linux/kthread.h>
  95 #include <linux/bpf.h>
  96 #include <linux/bpf_trace.h>
  97 #include <net/net_namespace.h>
  98 #include <net/sock.h>
  99 #include <net/busy_poll.h>
 100 #include <linux/rtnetlink.h>
 101 #include <linux/stat.h>
 102 #include <net/dsa.h>
 103 #include <net/dst.h>
 104 #include <net/dst_metadata.h>
 105 #include <net/gro.h>
 106 #include <net/pkt_sched.h>
 107 #include <net/pkt_cls.h>
 108 #include <net/checksum.h>
 109 #include <net/xfrm.h>
 110 #include <linux/highmem.h>
 111 #include <linux/init.h>
 112 #include <linux/module.h>
 113 #include <linux/netpoll.h>
 114 #include <linux/rcupdate.h>
 115 #include <linux/delay.h>
 116 #include <net/iw_handler.h>
 117 #include <asm/current.h>
 118 #include <linux/audit.h>
 119 #include <linux/dmaengine.h>
 120 #include <linux/err.h>
 121 #include <linux/ctype.h>
 122 #include <linux/if_arp.h>
 123 #include <linux/if_vlan.h>
 124 #include <linux/ip.h>
 125 #include <net/ip.h>
 126 #include <net/mpls.h>
 127 #include <linux/ipv6.h>
 128 #include <linux/in.h>
 129 #include <linux/jhash.h>
 130 #include <linux/random.h>
 131 #include <trace/events/napi.h>
 132 #include <trace/events/net.h>
 133 #include <trace/events/skb.h>
 134 #include <trace/events/qdisc.h>
 135 #include <linux/inetdevice.h>
 136 #include <linux/cpu_rmap.h>
 137 #include <linux/static_key.h>
 138 #include <linux/hashtable.h>
 139 #include <linux/vmalloc.h>
 140 #include <linux/if_macvlan.h>
 141 #include <linux/errqueue.h>
 142 #include <linux/hrtimer.h>
 143 #include <linux/netfilter_netdev.h>
 144 #include <linux/crash_dump.h>
 145 #include <linux/sctp.h>
 146 #include <net/udp_tunnel.h>
 147 #include <linux/net_namespace.h>
 148 #include <linux/indirect_call_wrapper.h>
 149 #include <net/devlink.h>
 150 #include <linux/pm_runtime.h>
 151 #include <linux/prandom.h>
 152 #include <linux/once_lite.h>
 153
 154 #include "dev.h"
 155 #include "net-sysfs.h"
 156
 157
 158 static DEFINE_SPINLOCK(ptype_lock);
 159 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 160 struct list_head ptype_all __read_mostly;       /* Taps */
 161
 162 static int netif_rx_internal(struct sk_buff *skb);
 163 static int call_netdevice_notifiers_info(unsigned long val,
 164                                          struct netdev_notifier_info *info);
 165 static int call_netdevice_notifiers_extack(unsigned long val,
 166                                            struct net_device *dev,
 167                                            struct netlink_ext_ack *extack);
 168 static struct napi_struct *napi_by_id(unsigned int napi_id);
 169
 170 /*
 171  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
 172  * semaphore.
 173  *
 174  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
 175  *
 176  * Writers must hold the rtnl semaphore while they loop through the
 177  * dev_base_head list, and hold dev_base_lock for writing when they do the
 178  * actual updates.  This allows pure readers to access the list even
 179  * while a writer is preparing to update it.
 180  *
 181  * To put it another way, dev_base_lock is held for writing only to
 182  * protect against pure readers; the rtnl semaphore provides the
 183  * protection against other writers.
 184  *
 185  * See, for example usages, register_netdevice() and
 186  * unregister_netdevice(), which must be called with the rtnl
 187  * semaphore held.
 188  */
 189 DEFINE_RWLOCK(dev_base_lock);
 190 EXPORT_SYMBOL(dev_base_lock);
 191
 192 static DEFINE_MUTEX(ifalias_mutex);
 193
 194 /* protects napi_hash addition/deletion and napi_gen_id */
 195 static DEFINE_SPINLOCK(napi_hash_lock);
 196
 197 static unsigned int napi_gen_id = NR_CPUS;
 198 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
 199
 200 static DECLARE_RWSEM(devnet_rename_sem);
 201
 202 static inline void dev_base_seq_inc(struct net *net)
 203 {
 204         while (++net->dev_base_seq == 0)
 205                 ;
 206 }
 207
 208 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
 209 {
 210         unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
 211
 212         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
 213 }
 214
 215 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
 216 {
 217         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
 218 }
 219
 220 static inline void rps_lock_irqsave(struct softnet_data *sd,
 221                                     unsigned long *flags)
 222 {
 223         if (IS_ENABLED(CONFIG_RPS))
 224                 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
 225         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 226                 local_irq_save(*flags);
 227 }
 228
 229 static inline void rps_lock_irq_disable(struct softnet_data *sd)
 230 {
 231         if (IS_ENABLED(CONFIG_RPS))
 232                 spin_lock_irq(&sd->input_pkt_queue.lock);
 233         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 234                 local_irq_disable();
 235 }
 236
 237 static inline void rps_unlock_irq_restore(struct softnet_data *sd,
 238                                           unsigned long *flags)
 239 {
 240         if (IS_ENABLED(CONFIG_RPS))
 241                 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
 242         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 243                 local_irq_restore(*flags);
 244 }
 245
 246 static inline void rps_unlock_irq_enable(struct softnet_data *sd)
 247 {
 248         if (IS_ENABLED(CONFIG_RPS))
 249                 spin_unlock_irq(&sd->input_pkt_queue.lock);
 250         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 251                 local_irq_enable();
 252 }
 253
 254 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
 255                                                        const char *name)
 256 {
 257         struct netdev_name_node *name_node;
 258
 259         name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
 260         if (!name_node)
 261                 return NULL;
 262         INIT_HLIST_NODE(&name_node->hlist);
 263         name_node->dev = dev;
 264         name_node->name = name;
 265         return name_node;
 266 }
 267
 268 static struct netdev_name_node *
 269 netdev_name_node_head_alloc(struct net_device *dev)
 270 {
 271         struct netdev_name_node *name_node;
 272
 273         name_node = netdev_name_node_alloc(dev, dev->name);
 274         if (!name_node)
 275                 return NULL;
 276         INIT_LIST_HEAD(&name_node->list);
 277         return name_node;
 278 }
 279
 280 static void netdev_name_node_free(struct netdev_name_node *name_node)
 281 {
 282         kfree(name_node);
 283 }
 284
 285 static void netdev_name_node_add(struct net *net,
 286                                  struct netdev_name_node *name_node)
 287 {
 288         hlist_add_head_rcu(&name_node->hlist,
 289                            dev_name_hash(net, name_node->name));
 290 }
 291
 292 static void netdev_name_node_del(struct netdev_name_node *name_node)
 293 {
 294         hlist_del_rcu(&name_node->hlist);
 295 }
 296
 297 static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
 298                                                         const char *name)
 299 {
 300         struct hlist_head *head = dev_name_hash(net, name);
 301         struct netdev_name_node *name_node;
 302
 303         hlist_for_each_entry(name_node, head, hlist)
 304                 if (!strcmp(name_node->name, name))
 305                         return name_node;
 306         return NULL;
 307 }
 308
 309 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
 310                                                             const char *name)
 311 {
 312         struct hlist_head *head = dev_name_hash(net, name);
 313         struct netdev_name_node *name_node;
 314
 315         hlist_for_each_entry_rcu(name_node, head, hlist)
 316                 if (!strcmp(name_node->name, name))
 317                         return name_node;
 318         return NULL;
 319 }
 320
 321 bool netdev_name_in_use(struct net *net, const char *name)
 322 {
 323         return netdev_name_node_lookup(net, name);
 324 }
 325 EXPORT_SYMBOL(netdev_name_in_use);
 326
 327 int netdev_name_node_alt_create(struct net_device *dev, const char *name)
 328 {
 329         struct netdev_name_node *name_node;
 330         struct net *net = dev_net(dev);
 331
 332         name_node = netdev_name_node_lookup(net, name);
 333         if (name_node)
 334                 return -EEXIST;
 335         name_node = netdev_name_node_alloc(dev, name);
 336         if (!name_node)
 337                 return -ENOMEM;
 338         netdev_name_node_add(net, name_node);
 339         /* The node that holds dev->name acts as a head of per-device list. */
 340         list_add_tail(&name_node->list, &dev->name_node->list);
 341
 342         return 0;
 343 }
 344
 345 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
 346 {
 347         list_del(&name_node->list);
 348         netdev_name_node_del(name_node);
 349         kfree(name_node->name);
 350         netdev_name_node_free(name_node);
 351 }
 352
 353 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
 354 {
 355         struct netdev_name_node *name_node;
 356         struct net *net = dev_net(dev);
 357
 358         name_node = netdev_name_node_lookup(net, name);
 359         if (!name_node)
 360                 return -ENOENT;
 361         /* lookup might have found our primary name or a name belonging
 362          * to another device.
 363          */
 364         if (name_node == dev->name_node || name_node->dev != dev)
 365                 return -EINVAL;
 366
 367         __netdev_name_node_alt_destroy(name_node);
 368
 369         return 0;
 370 }
 371
 372 static void netdev_name_node_alt_flush(struct net_device *dev)
 373 {
 374         struct netdev_name_node *name_node, *tmp;
 375
 376         list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
 377                 __netdev_name_node_alt_destroy(name_node);
 378 }
 379
 380 /* Device list insertion */
 381 static void list_netdevice(struct net_device *dev)
 382 {
 383         struct net *net = dev_net(dev);
 384
 385         ASSERT_RTNL();
 386
 387         write_lock(&dev_base_lock);
 388         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
 389         netdev_name_node_add(net, dev->name_node);
 390         hlist_add_head_rcu(&dev->index_hlist,
 391                            dev_index_hash(net, dev->ifindex));
 392         write_unlock(&dev_base_lock);
 393
 394         dev_base_seq_inc(net);
 395 }
 396
 397 /* Device list removal
 398  * caller must respect a RCU grace period before freeing/reusing dev
 399  */
 400 static void unlist_netdevice(struct net_device *dev, bool lock)
 401 {
 402         ASSERT_RTNL();
 403
 404         /* Unlink dev from the device chain */
 405         if (lock)
 406                 write_lock(&dev_base_lock);
 407         list_del_rcu(&dev->dev_list);
 408         netdev_name_node_del(dev->name_node);
 409         hlist_del_rcu(&dev->index_hlist);
 410         if (lock)
 411                 write_unlock(&dev_base_lock);
 412
 413         dev_base_seq_inc(dev_net(dev));
 414 }
 415
 416 /*
 417  *      Our notifier list
 418  */
 419
 420 static RAW_NOTIFIER_HEAD(netdev_chain);
 421
 422 /*
 423  *      Device drivers call our routines to queue packets here. We empty the
 424  *      queue in the local softnet handler.
 425  */
 426
 427 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 428 EXPORT_PER_CPU_SYMBOL(softnet_data);
 429
 430 #ifdef CONFIG_LOCKDEP
 431 /*
 432  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
 433  * according to dev->type
 434  */
 435 static const unsigned short netdev_lock_type[] = {
 436          ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
 437          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
 438          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
 439          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
 440          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
 441          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
 442          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
 443          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
 444          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
 445          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
 446          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
 447          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
 448          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
 449          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
 450          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
 451
 452 static const char *const netdev_lock_name[] = {
 453         "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
 454         "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
 455         "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
 456         "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
 457         "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
 458         "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
 459         "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
 460         "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
 461         "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
 462         "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
 463         "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
 464         "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
 465         "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
 466         "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
 467         "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
 468
 469 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
 470 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
 471
 472 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
 473 {
 474         int i;
 475
 476         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
 477                 if (netdev_lock_type[i] == dev_type)
 478                         return i;
 479         /* the last key is used by default */
 480         return ARRAY_SIZE(netdev_lock_type) - 1;
 481 }
 482
 483 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 484                                                  unsigned short dev_type)
 485 {
 486         int i;
 487
 488         i = netdev_lock_pos(dev_type);
 489         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
 490                                    netdev_lock_name[i]);
 491 }
 492
 493 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 494 {
 495         int i;
 496
 497         i = netdev_lock_pos(dev->type);
 498         lockdep_set_class_and_name(&dev->addr_list_lock,
 499                                    &netdev_addr_lock_key[i],
 500                                    netdev_lock_name[i]);
 501 }
 502 #else
 503 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 504                                                  unsigned short dev_type)
 505 {
 506 }
 507
 508 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 509 {
 510 }
 511 #endif
 512
 513 /*******************************************************************************
 514  *
 515  *              Protocol management and registration routines
 516  *
 517  *******************************************************************************/
 518
 519
 520 /*
 521  *      Add a protocol ID to the list. Now that the input handler is
 522  *      smarter we can dispense with all the messy stuff that used to be
 523  *      here.
 524  *
 525  *      BEWARE!!! Protocol handlers, mangling input packets,
 526  *      MUST BE last in hash buckets and checking protocol handlers
 527  *      MUST start from promiscuous ptype_all chain in net_bh.
 528  *      It is true now, do not change it.
 529  *      Explanation follows: if protocol handler, mangling packet, will
 530  *      be the first on list, it is not able to sense, that packet
 531  *      is cloned and should be copied-on-write, so that it will
 532  *      change it and subsequent readers will get broken packet.
 533  *                                                      --ANK (980803)
 534  */
 535
 536 static inline struct list_head *ptype_head(const struct packet_type *pt)
 537 {
 538         if (pt->type == htons(ETH_P_ALL))
 539                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
 540         else
 541                 return pt->dev ? &pt->dev->ptype_specific :
 542                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
 543 }
 544
 545 /**
 546  *      dev_add_pack - add packet handler
 547  *      @pt: packet type declaration
 548  *
 549  *      Add a protocol handler to the networking stack. The passed &packet_type
 550  *      is linked into kernel lists and may not be freed until it has been
 551  *      removed from the kernel lists.
 552  *
 553  *      This call does not sleep therefore it can not
 554  *      guarantee all CPU's that are in middle of receiving packets
 555  *      will see the new packet type (until the next received packet).
 556  */
 557
 558 void dev_add_pack(struct packet_type *pt)
 559 {
 560         struct list_head *head = ptype_head(pt);
 561
 562         spin_lock(&ptype_lock);
 563         list_add_rcu(&pt->list, head);
 564         spin_unlock(&ptype_lock);
 565 }
 566 EXPORT_SYMBOL(dev_add_pack);
 567
 568 /**
 569  *      __dev_remove_pack        - remove packet handler
 570  *      @pt: packet type declaration
 571  *
 572  *      Remove a protocol handler that was previously added to the kernel
 573  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 574  *      from the kernel lists and can be freed or reused once this function
 575  *      returns.
 576  *
 577  *      The packet type might still be in use by receivers
 578  *      and must not be freed until after all the CPU's have gone
 579  *      through a quiescent state.
 580  */
 581 void __dev_remove_pack(struct packet_type *pt)
 582 {
 583         struct list_head *head = ptype_head(pt);
 584         struct packet_type *pt1;
 585
 586         spin_lock(&ptype_lock);
 587
 588         list_for_each_entry(pt1, head, list) {
 589                 if (pt == pt1) {
 590                         list_del_rcu(&pt->list);
 591                         goto out;
 592                 }
 593         }
 594
 595         pr_warn("dev_remove_pack: %p not found\n", pt);
 596 out:
 597         spin_unlock(&ptype_lock);
 598 }
 599 EXPORT_SYMBOL(__dev_remove_pack);
 600
 601 /**
 602  *      dev_remove_pack  - remove packet handler
 603  *      @pt: packet type declaration
 604  *
 605  *      Remove a protocol handler that was previously added to the kernel
 606  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 607  *      from the kernel lists and can be freed or reused once this function
 608  *      returns.
 609  *
 610  *      This call sleeps to guarantee that no CPU is looking at the packet
 611  *      type after return.
 612  */
 613 void dev_remove_pack(struct packet_type *pt)
 614 {
 615         __dev_remove_pack(pt);
 616
 617         synchronize_net();
 618 }
 619 EXPORT_SYMBOL(dev_remove_pack);
 620
 621
 622 /*******************************************************************************
 623  *
 624  *                          Device Interface Subroutines
 625  *
 626  *******************************************************************************/
 627
 628 /**
 629  *      dev_get_iflink  - get 'iflink' value of a interface
 630  *      @dev: targeted interface
 631  *
 632  *      Indicates the ifindex the interface is linked to.
 633  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
 634  */
 635
 636 int dev_get_iflink(const struct net_device *dev)
 637 {
 638         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
 639                 return dev->netdev_ops->ndo_get_iflink(dev);
 640
 641         return dev->ifindex;
 642 }
 643 EXPORT_SYMBOL(dev_get_iflink);
 644
 645 /**
 646  *      dev_fill_metadata_dst - Retrieve tunnel egress information.
 647  *      @dev: targeted interface
 648  *      @skb: The packet.
 649  *
 650  *      For better visibility of tunnel traffic OVS needs to retrieve
 651  *      egress tunnel information for a packet. Following API allows
 652  *      user to get this info.
 653  */
 654 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 655 {
 656         struct ip_tunnel_info *info;
 657
 658         if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
 659                 return -EINVAL;
 660
 661         info = skb_tunnel_info_unclone(skb);
 662         if (!info)
 663                 return -ENOMEM;
 664         if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
 665                 return -EINVAL;
 666
 667         return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
 668 }
 669 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
 670
 671 static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
 672 {
 673         int k = stack->num_paths++;
 674
 675         if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
 676                 return NULL;
 677
 678         return &stack->path[k];
 679 }
 680
 681 int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
 682                           struct net_device_path_stack *stack)
 683 {
 684         const struct net_device *last_dev;
 685         struct net_device_path_ctx ctx = {
 686                 .dev    = dev,
 687         };
 688         struct net_device_path *path;
 689         int ret = 0;
 690
 691         memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
 692         stack->num_paths = 0;
 693         while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
 694                 last_dev = ctx.dev;
 695                 path = dev_fwd_path(stack);
 696                 if (!path)
 697                         return -1;
 698
 699                 memset(path, 0, sizeof(struct net_device_path));
 700                 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
 701                 if (ret < 0)
 702                         return -1;
 703
 704                 if (WARN_ON_ONCE(last_dev == ctx.dev))
 705                         return -1;
 706         }
 707
 708         if (!ctx.dev)
 709                 return ret;
 710
 711         path = dev_fwd_path(stack);
 712         if (!path)
 713                 return -1;
 714         path->type = DEV_PATH_ETHERNET;
 715         path->dev = ctx.dev;
 716
 717         return ret;
 718 }
 719 EXPORT_SYMBOL_GPL(dev_fill_forward_path);
 720
 721 /**
 722  *      __dev_get_by_name       - find a device by its name
 723  *      @net: the applicable net namespace
 724  *      @name: name to find
 725  *
 726  *      Find an interface by name. Must be called under RTNL semaphore
 727  *      or @dev_base_lock. If the name is found a pointer to the device
 728  *      is returned. If the name is not found then %NULL is returned. The
 729  *      reference counters are not incremented so the caller must be
 730  *      careful with locks.
 731  */
 732
 733 struct net_device *__dev_get_by_name(struct net *net, const char *name)
 734 {
 735         struct netdev_name_node *node_name;
 736
 737         node_name = netdev_name_node_lookup(net, name);
 738         return node_name ? node_name->dev : NULL;
 739 }
 740 EXPORT_SYMBOL(__dev_get_by_name);
 741
 742 /**
 743  * dev_get_by_name_rcu  - find a device by its name
 744  * @net: the applicable net namespace
 745  * @name: name to find
 746  *
 747  * Find an interface by name.
 748  * If the name is found a pointer to the device is returned.
 749  * If the name is not found then %NULL is returned.
 750  * The reference counters are not incremented so the caller must be
 751  * careful with locks. The caller must hold RCU lock.
 752  */
 753
 754 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
 755 {
 756         struct netdev_name_node *node_name;
 757
 758         node_name = netdev_name_node_lookup_rcu(net, name);
 759         return node_name ? node_name->dev : NULL;
 760 }
 761 EXPORT_SYMBOL(dev_get_by_name_rcu);
 762
 763 /**
 764  *      dev_get_by_name         - find a device by its name
 765  *      @net: the applicable net namespace
 766  *      @name: name to find
 767  *
 768  *      Find an interface by name. This can be called from any
 769  *      context and does its own locking. The returned handle has
 770  *      the usage count incremented and the caller must use dev_put() to
 771  *      release it when it is no longer needed. %NULL is returned if no
 772  *      matching device is found.
 773  */
 774
 775 struct net_device *dev_get_by_name(struct net *net, const char *name)
 776 {
 777         struct net_device *dev;
 778
 779         rcu_read_lock();
 780         dev = dev_get_by_name_rcu(net, name);
 781         dev_hold(dev);
 782         rcu_read_unlock();
 783         return dev;
 784 }
 785 EXPORT_SYMBOL(dev_get_by_name);
 786
 787 /**
 788  *      __dev_get_by_index - find a device by its ifindex
 789  *      @net: the applicable net namespace
 790  *      @ifindex: index of device
 791  *
 792  *      Search for an interface by index. Returns %NULL if the device
 793  *      is not found or a pointer to the device. The device has not
 794  *      had its reference counter increased so the caller must be careful
 795  *      about locking. The caller must hold either the RTNL semaphore
 796  *      or @dev_base_lock.
 797  */
 798
 799 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
 800 {
 801         struct net_device *dev;
 802         struct hlist_head *head = dev_index_hash(net, ifindex);
 803
 804         hlist_for_each_entry(dev, head, index_hlist)
 805                 if (dev->ifindex == ifindex)
 806                         return dev;
 807
 808         return NULL;
 809 }
 810 EXPORT_SYMBOL(__dev_get_by_index);
 811
 812 /**
 813  *      dev_get_by_index_rcu - find a device by its ifindex
 814  *      @net: the applicable net namespace
 815  *      @ifindex: index of device
 816  *
 817  *      Search for an interface by index. Returns %NULL if the device
 818  *      is not found or a pointer to the device. The device has not
 819  *      had its reference counter increased so the caller must be careful
 820  *      about locking. The caller must hold RCU lock.
 821  */
 822
 823 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
 824 {
 825         struct net_device *dev;
 826         struct hlist_head *head = dev_index_hash(net, ifindex);
 827
 828         hlist_for_each_entry_rcu(dev, head, index_hlist)
 829                 if (dev->ifindex == ifindex)
 830                         return dev;
 831
 832         return NULL;
 833 }
 834 EXPORT_SYMBOL(dev_get_by_index_rcu);
 835
 836
 837 /**
 838  *      dev_get_by_index - find a device by its ifindex
 839  *      @net: the applicable net namespace
 840  *      @ifindex: index of device
 841  *
 842  *      Search for an interface by index. Returns NULL if the device
 843  *      is not found or a pointer to the device. The device returned has
 844  *      had a reference added and the pointer is safe until the user calls
 845  *      dev_put to indicate they have finished with it.
 846  */
 847
 848 struct net_device *dev_get_by_index(struct net *net, int ifindex)
 849 {
 850         struct net_device *dev;
 851
 852         rcu_read_lock();
 853         dev = dev_get_by_index_rcu(net, ifindex);
 854         dev_hold(dev);
 855         rcu_read_unlock();
 856         return dev;
 857 }
 858 EXPORT_SYMBOL(dev_get_by_index);
 859
 860 /**
 861  *      dev_get_by_napi_id - find a device by napi_id
 862  *      @napi_id: ID of the NAPI struct
 863  *
 864  *      Search for an interface by NAPI ID. Returns %NULL if the device
 865  *      is not found or a pointer to the device. The device has not had
 866  *      its reference counter increased so the caller must be careful
 867  *      about locking. The caller must hold RCU lock.
 868  */
 869
 870 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
 871 {
 872         struct napi_struct *napi;
 873
 874         WARN_ON_ONCE(!rcu_read_lock_held());
 875
 876         if (napi_id < MIN_NAPI_ID)
 877                 return NULL;
 878
 879         napi = napi_by_id(napi_id);
 880
 881         return napi ? napi->dev : NULL;
 882 }
 883 EXPORT_SYMBOL(dev_get_by_napi_id);
 884
 885 /**
 886  *      netdev_get_name - get a netdevice name, knowing its ifindex.
 887  *      @net: network namespace
 888  *      @name: a pointer to the buffer where the name will be stored.
 889  *      @ifindex: the ifindex of the interface to get the name from.
 890  */
 891 int netdev_get_name(struct net *net, char *name, int ifindex)
 892 {
 893         struct net_device *dev;
 894         int ret;
 895
 896         down_read(&devnet_rename_sem);
 897         rcu_read_lock();
 898
 899         dev = dev_get_by_index_rcu(net, ifindex);
 900         if (!dev) {
 901                 ret = -ENODEV;
 902                 goto out;
 903         }
 904
 905         strcpy(name, dev->name);
 906
 907         ret = 0;
 908 out:
 909         rcu_read_unlock();
 910         up_read(&devnet_rename_sem);
 911         return ret;
 912 }
 913
 914 /**
 915  *      dev_getbyhwaddr_rcu - find a device by its hardware address
 916  *      @net: the applicable net namespace
 917  *      @type: media type of device
 918  *      @ha: hardware address
 919  *
 920  *      Search for an interface by MAC address. Returns NULL if the device
 921  *      is not found or a pointer to the device.
 922  *      The caller must hold RCU or RTNL.
 923  *      The returned device has not had its ref count increased
 924  *      and the caller must therefore be careful about locking
 925  *
 926  */
 927
 928 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
 929                                        const char *ha)
 930 {
 931         struct net_device *dev;
 932
 933         for_each_netdev_rcu(net, dev)
 934                 if (dev->type == type &&
 935                     !memcmp(dev->dev_addr, ha, dev->addr_len))
 936                         return dev;
 937
 938         return NULL;
 939 }
 940 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
 941
 942 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
 943 {
 944         struct net_device *dev, *ret = NULL;
 945
 946         rcu_read_lock();
 947         for_each_netdev_rcu(net, dev)
 948                 if (dev->type == type) {
 949                         dev_hold(dev);
 950                         ret = dev;
 951                         break;
 952                 }
 953         rcu_read_unlock();
 954         return ret;
 955 }
 956 EXPORT_SYMBOL(dev_getfirstbyhwtype);
 957
 958 /**
 959  *      __dev_get_by_flags - find any device with given flags
 960  *      @net: the applicable net namespace
 961  *      @if_flags: IFF_* values
 962  *      @mask: bitmask of bits in if_flags to check
 963  *
 964  *      Search for any interface with the given flags. Returns NULL if a device
 965  *      is not found or a pointer to the device. Must be called inside
 966  *      rtnl_lock(), and result refcount is unchanged.
 967  */
 968
 969 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
 970                                       unsigned short mask)
 971 {
 972         struct net_device *dev, *ret;
 973
 974         ASSERT_RTNL();
 975
 976         ret = NULL;
 977         for_each_netdev(net, dev) {
 978                 if (((dev->flags ^ if_flags) & mask) == 0) {
 979                         ret = dev;
 980                         break;
 981                 }
 982         }
 983         return ret;
 984 }
 985 EXPORT_SYMBOL(__dev_get_by_flags);
 986
 987 /**
 988  *      dev_valid_name - check if name is okay for network device
 989  *      @name: name string
 990  *
 991  *      Network device names need to be valid file names to
 992  *      allow sysfs to work.  We also disallow any kind of
 993  *      whitespace.
 994  */
 995 bool dev_valid_name(const char *name)
 996 {
 997         if (*name == '\0')
 998                 return false;
 999         if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1000                 return false;
1001         if (!strcmp(name, ".") || !strcmp(name, ".."))
1002                 return false;
1003
1004         while (*name) {
1005                 if (*name == '/' || *name == ':' || isspace(*name))
1006                         return false;
1007                 name++;
1008         }
1009         return true;
1010 }
1011 EXPORT_SYMBOL(dev_valid_name);
1012
1013 /**
1014  *      __dev_alloc_name - allocate a name for a device
1015  *      @net: network namespace to allocate the device name in
1016  *      @name: name format string
1017  *      @buf:  scratch buffer and result name string
1018  *
1019  *      Passed a format string - eg "lt%d" it will try and find a suitable
1020  *      id. It scans list of devices to build up a free map, then chooses
1021  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1022  *      while allocating the name and adding the device in order to avoid
1023  *      duplicates.
1024  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1025  *      Returns the number of the unit assigned or a negative errno code.
1026  */
1027
1028 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1029 {
1030         int i = 0;
1031         const char *p;
1032         const int max_netdevices = 8*PAGE_SIZE;
1033         unsigned long *inuse;
1034         struct net_device *d;
1035
1036         if (!dev_valid_name(name))
1037                 return -EINVAL;
1038
1039         p = strchr(name, '%');
1040         if (p) {
1041                 /*
1042                  * Verify the string as this thing may have come from
1043                  * the user.  There must be either one "%d" and no other "%"
1044                  * characters.
1045                  */
1046                 if (p[1] != 'd' || strchr(p + 2, '%'))
1047                         return -EINVAL;
1048
1049                 /* Use one page as a bit array of possible slots */
1050                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1051                 if (!inuse)
1052                         return -ENOMEM;
1053
1054                 for_each_netdev(net, d) {
1055                         struct netdev_name_node *name_node;
1056                         list_for_each_entry(name_node, &d->name_node->list, list) {
1057                                 if (!sscanf(name_node->name, name, &i))
1058                                         continue;
1059                                 if (i < 0 || i >= max_netdevices)
1060                                         continue;
1061
1062                                 /*  avoid cases where sscanf is not exact inverse of printf */
1063                                 snprintf(buf, IFNAMSIZ, name, i);
1064                                 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1065                                         __set_bit(i, inuse);
1066                         }
1067                         if (!sscanf(d->name, name, &i))
1068                                 continue;
1069                         if (i < 0 || i >= max_netdevices)
1070                                 continue;
1071
1072                         /*  avoid cases where sscanf is not exact inverse of printf */
1073                         snprintf(buf, IFNAMSIZ, name, i);
1074                         if (!strncmp(buf, d->name, IFNAMSIZ))
1075                                 __set_bit(i, inuse);
1076                 }
1077
1078                 i = find_first_zero_bit(inuse, max_netdevices);
1079                 free_page((unsigned long) inuse);
1080         }
1081
1082         snprintf(buf, IFNAMSIZ, name, i);
1083         if (!netdev_name_in_use(net, buf))
1084                 return i;
1085
1086         /* It is possible to run out of possible slots
1087          * when the name is long and there isn't enough space left
1088          * for the digits, or if all bits are used.
1089          */
1090         return -ENFILE;
1091 }
1092
1093 static int dev_alloc_name_ns(struct net *net,
1094                              struct net_device *dev,
1095                              const char *name)
1096 {
1097         char buf[IFNAMSIZ];
1098         int ret;
1099
1100         BUG_ON(!net);
1101         ret = __dev_alloc_name(net, name, buf);
1102         if (ret >= 0)
1103                 strscpy(dev->name, buf, IFNAMSIZ);
1104         return ret;
1105 }
1106
1107 /**
1108  *      dev_alloc_name - allocate a name for a device
1109  *      @dev: device
1110  *      @name: name format string
1111  *
1112  *      Passed a format string - eg "lt%d" it will try and find a suitable
1113  *      id. It scans list of devices to build up a free map, then chooses
1114  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1115  *      while allocating the name and adding the device in order to avoid
1116  *      duplicates.
1117  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1118  *      Returns the number of the unit assigned or a negative errno code.
1119  */
1120
1121 int dev_alloc_name(struct net_device *dev, const char *name)
1122 {
1123         return dev_alloc_name_ns(dev_net(dev), dev, name);
1124 }
1125 EXPORT_SYMBOL(dev_alloc_name);
1126
1127 static int dev_get_valid_name(struct net *net, struct net_device *dev,
1128                               const char *name)
1129 {
1130         BUG_ON(!net);
1131
1132         if (!dev_valid_name(name))
1133                 return -EINVAL;
1134
1135         if (strchr(name, '%'))
1136                 return dev_alloc_name_ns(net, dev, name);
1137         else if (netdev_name_in_use(net, name))
1138                 return -EEXIST;
1139         else if (dev->name != name)
1140                 strscpy(dev->name, name, IFNAMSIZ);
1141
1142         return 0;
1143 }
1144
1145 /**
1146  *      dev_change_name - change name of a device
1147  *      @dev: device
1148  *      @newname: name (or format string) must be at least IFNAMSIZ
1149  *
1150  *      Change name of a device, can pass format strings "eth%d".
1151  *      for wildcarding.
1152  */
1153 int dev_change_name(struct net_device *dev, const char *newname)
1154 {
1155         unsigned char old_assign_type;
1156         char oldname[IFNAMSIZ];
1157         int err = 0;
1158         int ret;
1159         struct net *net;
1160
1161         ASSERT_RTNL();
1162         BUG_ON(!dev_net(dev));
1163
1164         net = dev_net(dev);
1165
1166         down_write(&devnet_rename_sem);
1167
1168         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1169                 up_write(&devnet_rename_sem);
1170                 return 0;
1171         }
1172
1173         memcpy(oldname, dev->name, IFNAMSIZ);
1174
1175         err = dev_get_valid_name(net, dev, newname);
1176         if (err < 0) {
1177                 up_write(&devnet_rename_sem);
1178                 return err;
1179         }
1180
1181         if (oldname[0] && !strchr(oldname, '%'))
1182                 netdev_info(dev, "renamed from %s%s\n", oldname,
1183                             dev->flags & IFF_UP ? " (while UP)" : "");
1184
1185         old_assign_type = dev->name_assign_type;
1186         dev->name_assign_type = NET_NAME_RENAMED;
1187
1188 rollback:
1189         ret = device_rename(&dev->dev, dev->name);
1190         if (ret) {
1191                 memcpy(dev->name, oldname, IFNAMSIZ);
1192                 dev->name_assign_type = old_assign_type;
1193                 up_write(&devnet_rename_sem);
1194                 return ret;
1195         }
1196
1197         up_write(&devnet_rename_sem);
1198
1199         netdev_adjacent_rename_links(dev, oldname);
1200
1201         write_lock(&dev_base_lock);
1202         netdev_name_node_del(dev->name_node);
1203         write_unlock(&dev_base_lock);
1204
1205         synchronize_rcu();
1206
1207         write_lock(&dev_base_lock);
1208         netdev_name_node_add(net, dev->name_node);
1209         write_unlock(&dev_base_lock);
1210
1211         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1212         ret = notifier_to_errno(ret);
1213
1214         if (ret) {
1215                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1216                 if (err >= 0) {
1217                         err = ret;
1218                         down_write(&devnet_rename_sem);
1219                         memcpy(dev->name, oldname, IFNAMSIZ);
1220                         memcpy(oldname, newname, IFNAMSIZ);
1221                         dev->name_assign_type = old_assign_type;
1222                         old_assign_type = NET_NAME_RENAMED;
1223                         goto rollback;
1224                 } else {
1225                         netdev_err(dev, "name change rollback failed: %d\n",
1226                                    ret);
1227                 }
1228         }
1229
1230         return err;
1231 }
1232
1233 /**
1234  *      dev_set_alias - change ifalias of a device
1235  *      @dev: device
1236  *      @alias: name up to IFALIASZ
1237  *      @len: limit of bytes to copy from info
1238  *
1239  *      Set ifalias for a device,
1240  */
1241 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1242 {
1243         struct dev_ifalias *new_alias = NULL;
1244
1245         if (len >= IFALIASZ)
1246                 return -EINVAL;
1247
1248         if (len) {
1249                 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1250                 if (!new_alias)
1251                         return -ENOMEM;
1252
1253                 memcpy(new_alias->ifalias, alias, len);
1254                 new_alias->ifalias[len] = 0;
1255         }
1256
1257         mutex_lock(&ifalias_mutex);
1258         new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1259                                         mutex_is_locked(&ifalias_mutex));
1260         mutex_unlock(&ifalias_mutex);
1261
1262         if (new_alias)
1263                 kfree_rcu(new_alias, rcuhead);
1264
1265         return len;
1266 }
1267 EXPORT_SYMBOL(dev_set_alias);
1268
1269 /**
1270  *      dev_get_alias - get ifalias of a device
1271  *      @dev: device
1272  *      @name: buffer to store name of ifalias
1273  *      @len: size of buffer
1274  *
1275  *      get ifalias for a device.  Caller must make sure dev cannot go
1276  *      away,  e.g. rcu read lock or own a reference count to device.
1277  */
1278 int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1279 {
1280         const struct dev_ifalias *alias;
1281         int ret = 0;
1282
1283         rcu_read_lock();
1284         alias = rcu_dereference(dev->ifalias);
1285         if (alias)
1286                 ret = snprintf(name, len, "%s", alias->ifalias);
1287         rcu_read_unlock();
1288
1289         return ret;
1290 }
1291
1292 /**
1293  *      netdev_features_change - device changes features
1294  *      @dev: device to cause notification
1295  *
1296  *      Called to indicate a device has changed features.
1297  */
1298 void netdev_features_change(struct net_device *dev)
1299 {
1300         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1301 }
1302 EXPORT_SYMBOL(netdev_features_change);
1303
1304 /**
1305  *      netdev_state_change - device changes state
1306  *      @dev: device to cause notification
1307  *
1308  *      Called to indicate a device has changed state. This function calls
1309  *      the notifier chains for netdev_chain and sends a NEWLINK message
1310  *      to the routing socket.
1311  */
1312 void netdev_state_change(struct net_device *dev)
1313 {
1314         if (dev->flags & IFF_UP) {
1315                 struct netdev_notifier_change_info change_info = {
1316                         .info.dev = dev,
1317                 };
1318
1319                 call_netdevice_notifiers_info(NETDEV_CHANGE,
1320                                               &change_info.info);
1321                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL);
1322         }
1323 }
1324 EXPORT_SYMBOL(netdev_state_change);
1325
1326 /**
1327  * __netdev_notify_peers - notify network peers about existence of @dev,
1328  * to be called when rtnl lock is already held.
1329  * @dev: network device
1330  *
1331  * Generate traffic such that interested network peers are aware of
1332  * @dev, such as by generating a gratuitous ARP. This may be used when
1333  * a device wants to inform the rest of the network about some sort of
1334  * reconfiguration such as a failover event or virtual machine
1335  * migration.
1336  */
1337 void __netdev_notify_peers(struct net_device *dev)
1338 {
1339         ASSERT_RTNL();
1340         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1341         call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1342 }
1343 EXPORT_SYMBOL(__netdev_notify_peers);
1344
1345 /**
1346  * netdev_notify_peers - notify network peers about existence of @dev
1347  * @dev: network device
1348  *
1349  * Generate traffic such that interested network peers are aware of
1350  * @dev, such as by generating a gratuitous ARP. This may be used when
1351  * a device wants to inform the rest of the network about some sort of
1352  * reconfiguration such as a failover event or virtual machine
1353  * migration.
1354  */
1355 void netdev_notify_peers(struct net_device *dev)
1356 {
1357         rtnl_lock();
1358         __netdev_notify_peers(dev);
1359         rtnl_unlock();
1360 }
1361 EXPORT_SYMBOL(netdev_notify_peers);
1362
1363 static int napi_threaded_poll(void *data);
1364
1365 static int napi_kthread_create(struct napi_struct *n)
1366 {
1367         int err = 0;
1368
1369         /* Create and wake up the kthread once to put it in
1370          * TASK_INTERRUPTIBLE mode to avoid the blocked task
1371          * warning and work with loadavg.
1372          */
1373         n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1374                                 n->dev->name, n->napi_id);
1375         if (IS_ERR(n->thread)) {
1376                 err = PTR_ERR(n->thread);
1377                 pr_err("kthread_run failed with err %d\n", err);
1378                 n->thread = NULL;
1379         }
1380
1381         return err;
1382 }
1383
1384 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1385 {
1386         const struct net_device_ops *ops = dev->netdev_ops;
1387         int ret;
1388
1389         ASSERT_RTNL();
1390         dev_addr_check(dev);
1391
1392         if (!netif_device_present(dev)) {
1393                 /* may be detached because parent is runtime-suspended */
1394                 if (dev->dev.parent)
1395                         pm_runtime_resume(dev->dev.parent);
1396                 if (!netif_device_present(dev))
1397                         return -ENODEV;
1398         }
1399
1400         /* Block netpoll from trying to do any rx path servicing.
1401          * If we don't do this there is a chance ndo_poll_controller
1402          * or ndo_poll may be running while we open the device
1403          */
1404         netpoll_poll_disable(dev);
1405
1406         ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1407         ret = notifier_to_errno(ret);
1408         if (ret)
1409                 return ret;
1410
1411         set_bit(__LINK_STATE_START, &dev->state);
1412
1413         if (ops->ndo_validate_addr)
1414                 ret = ops->ndo_validate_addr(dev);
1415
1416         if (!ret && ops->ndo_open)
1417                 ret = ops->ndo_open(dev);
1418
1419         netpoll_poll_enable(dev);
1420
1421         if (ret)
1422                 clear_bit(__LINK_STATE_START, &dev->state);
1423         else {
1424                 dev->flags |= IFF_UP;
1425                 dev_set_rx_mode(dev);
1426                 dev_activate(dev);
1427                 add_device_randomness(dev->dev_addr, dev->addr_len);
1428         }
1429
1430         return ret;
1431 }
1432
1433 /**
1434  *      dev_open        - prepare an interface for use.
1435  *      @dev: device to open
1436  *      @extack: netlink extended ack
1437  *
1438  *      Takes a device from down to up state. The device's private open
1439  *      function is invoked and then the multicast lists are loaded. Finally
1440  *      the device is moved into the up state and a %NETDEV_UP message is
1441  *      sent to the netdev notifier chain.
1442  *
1443  *      Calling this function on an active interface is a nop. On a failure
1444  *      a negative errno code is returned.
1445  */
1446 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1447 {
1448         int ret;
1449
1450         if (dev->flags & IFF_UP)
1451                 return 0;
1452
1453         ret = __dev_open(dev, extack);
1454         if (ret < 0)
1455                 return ret;
1456
1457         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1458         call_netdevice_notifiers(NETDEV_UP, dev);
1459
1460         return ret;
1461 }
1462 EXPORT_SYMBOL(dev_open);
1463
1464 static void __dev_close_many(struct list_head *head)
1465 {
1466         struct net_device *dev;
1467
1468         ASSERT_RTNL();
1469         might_sleep();
1470
1471         list_for_each_entry(dev, head, close_list) {
1472                 /* Temporarily disable netpoll until the interface is down */
1473                 netpoll_poll_disable(dev);
1474
1475                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1476
1477                 clear_bit(__LINK_STATE_START, &dev->state);
1478
1479                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1480                  * can be even on different cpu. So just clear netif_running().
1481                  *
1482                  * dev->stop() will invoke napi_disable() on all of it's
1483                  * napi_struct instances on this device.
1484                  */
1485                 smp_mb__after_atomic(); /* Commit netif_running(). */
1486         }
1487
1488         dev_deactivate_many(head);
1489
1490         list_for_each_entry(dev, head, close_list) {
1491                 const struct net_device_ops *ops = dev->netdev_ops;
1492
1493                 /*
1494                  *      Call the device specific close. This cannot fail.
1495                  *      Only if device is UP
1496                  *
1497                  *      We allow it to be called even after a DETACH hot-plug
1498                  *      event.
1499                  */
1500                 if (ops->ndo_stop)
1501                         ops->ndo_stop(dev);
1502
1503                 dev->flags &= ~IFF_UP;
1504                 netpoll_poll_enable(dev);
1505         }
1506 }
1507
1508 static void __dev_close(struct net_device *dev)
1509 {
1510         LIST_HEAD(single);
1511
1512         list_add(&dev->close_list, &single);
1513         __dev_close_many(&single);
1514         list_del(&single);
1515 }
1516
1517 void dev_close_many(struct list_head *head, bool unlink)
1518 {
1519         struct net_device *dev, *tmp;
1520
1521         /* Remove the devices that don't need to be closed */
1522         list_for_each_entry_safe(dev, tmp, head, close_list)
1523                 if (!(dev->flags & IFF_UP))
1524                         list_del_init(&dev->close_list);
1525
1526         __dev_close_many(head);
1527
1528         list_for_each_entry_safe(dev, tmp, head, close_list) {
1529                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1530                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1531                 if (unlink)
1532                         list_del_init(&dev->close_list);
1533         }
1534 }
1535 EXPORT_SYMBOL(dev_close_many);
1536
1537 /**
1538  *      dev_close - shutdown an interface.
1539  *      @dev: device to shutdown
1540  *
1541  *      This function moves an active device into down state. A
1542  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1543  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1544  *      chain.
1545  */
1546 void dev_close(struct net_device *dev)
1547 {
1548         if (dev->flags & IFF_UP) {
1549                 LIST_HEAD(single);
1550
1551                 list_add(&dev->close_list, &single);
1552                 dev_close_many(&single, true);
1553                 list_del(&single);
1554         }
1555 }
1556 EXPORT_SYMBOL(dev_close);
1557
1558
1559 /**
1560  *      dev_disable_lro - disable Large Receive Offload on a device
1561  *      @dev: device
1562  *
1563  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1564  *      called under RTNL.  This is needed if received packets may be
1565  *      forwarded to another interface.
1566  */
1567 void dev_disable_lro(struct net_device *dev)
1568 {
1569         struct net_device *lower_dev;
1570         struct list_head *iter;
1571
1572         dev->wanted_features &= ~NETIF_F_LRO;
1573         netdev_update_features(dev);
1574
1575         if (unlikely(dev->features & NETIF_F_LRO))
1576                 netdev_WARN(dev, "failed to disable LRO!\n");
1577
1578         netdev_for_each_lower_dev(dev, lower_dev, iter)
1579                 dev_disable_lro(lower_dev);
1580 }
1581 EXPORT_SYMBOL(dev_disable_lro);
1582
1583 /**
1584  *      dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1585  *      @dev: device
1586  *
1587  *      Disable HW Generic Receive Offload (GRO_HW) on a net device.  Must be
1588  *      called under RTNL.  This is needed if Generic XDP is installed on
1589  *      the device.
1590  */
1591 static void dev_disable_gro_hw(struct net_device *dev)
1592 {
1593         dev->wanted_features &= ~NETIF_F_GRO_HW;
1594         netdev_update_features(dev);
1595
1596         if (unlikely(dev->features & NETIF_F_GRO_HW))
1597                 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1598 }
1599
1600 const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1601 {
1602 #define N(val)                                          \
1603         case NETDEV_##val:                              \
1604                 return "NETDEV_" __stringify(val);
1605         switch (cmd) {
1606         N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1607         N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1608         N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1609         N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN)
1610         N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA)
1611         N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE)
1612         N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1613         N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1614         N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1615         N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1616         N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1617         N(XDP_FEAT_CHANGE)
1618         }
1619 #undef N
1620         return "UNKNOWN_NETDEV_EVENT";
1621 }
1622 EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1623
1624 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1625                                    struct net_device *dev)
1626 {
1627         struct netdev_notifier_info info = {
1628                 .dev = dev,
1629         };
1630
1631         return nb->notifier_call(nb, val, &info);
1632 }
1633
1634 static int call_netdevice_register_notifiers(struct notifier_block *nb,
1635                                              struct net_device *dev)
1636 {
1637         int err;
1638
1639         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1640         err = notifier_to_errno(err);
1641         if (err)
1642                 return err;
1643
1644         if (!(dev->flags & IFF_UP))
1645                 return 0;
1646
1647         call_netdevice_notifier(nb, NETDEV_UP, dev);
1648         return 0;
1649 }
1650
1651 static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1652                                                 struct net_device *dev)
1653 {
1654         if (dev->flags & IFF_UP) {
1655                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1656                                         dev);
1657                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1658         }
1659         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1660 }
1661
1662 static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1663                                                  struct net *net)
1664 {
1665         struct net_device *dev;
1666         int err;
1667
1668         for_each_netdev(net, dev) {
1669                 err = call_netdevice_register_notifiers(nb, dev);
1670                 if (err)
1671                         goto rollback;
1672         }
1673         return 0;
1674
1675 rollback:
1676         for_each_netdev_continue_reverse(net, dev)
1677                 call_netdevice_unregister_notifiers(nb, dev);
1678         return err;
1679 }
1680
1681 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1682                                                     struct net *net)
1683 {
1684         struct net_device *dev;
1685
1686         for_each_netdev(net, dev)
1687                 call_netdevice_unregister_notifiers(nb, dev);
1688 }
1689
1690 static int dev_boot_phase = 1;
1691
1692 /**
1693  * register_netdevice_notifier - register a network notifier block
1694  * @nb: notifier
1695  *
1696  * Register a notifier to be called when network device events occur.
1697  * The notifier passed is linked into the kernel structures and must
1698  * not be reused until it has been unregistered. A negative errno code
1699  * is returned on a failure.
1700  *
1701  * When registered all registration and up events are replayed
1702  * to the new notifier to allow device to have a race free
1703  * view of the network device list.
1704  */
1705
1706 int register_netdevice_notifier(struct notifier_block *nb)
1707 {
1708         struct net *net;
1709         int err;
1710
1711         /* Close race with setup_net() and cleanup_net() */
1712         down_write(&pernet_ops_rwsem);
1713         rtnl_lock();
1714         err = raw_notifier_chain_register(&netdev_chain, nb);
1715         if (err)
1716                 goto unlock;
1717         if (dev_boot_phase)
1718                 goto unlock;
1719         for_each_net(net) {
1720                 err = call_netdevice_register_net_notifiers(nb, net);
1721                 if (err)
1722                         goto rollback;
1723         }
1724
1725 unlock:
1726         rtnl_unlock();
1727         up_write(&pernet_ops_rwsem);
1728         return err;
1729
1730 rollback:
1731         for_each_net_continue_reverse(net)
1732                 call_netdevice_unregister_net_notifiers(nb, net);
1733
1734         raw_notifier_chain_unregister(&netdev_chain, nb);
1735         goto unlock;
1736 }
1737 EXPORT_SYMBOL(register_netdevice_notifier);
1738
1739 /**
1740  * unregister_netdevice_notifier - unregister a network notifier block
1741  * @nb: notifier
1742  *
1743  * Unregister a notifier previously registered by
1744  * register_netdevice_notifier(). The notifier is unlinked into the
1745  * kernel structures and may then be reused. A negative errno code
1746  * is returned on a failure.
1747  *
1748  * After unregistering unregister and down device events are synthesized
1749  * for all devices on the device list to the removed notifier to remove
1750  * the need for special case cleanup code.
1751  */
1752
1753 int unregister_netdevice_notifier(struct notifier_block *nb)
1754 {
1755         struct net *net;
1756         int err;
1757
1758         /* Close race with setup_net() and cleanup_net() */
1759         down_write(&pernet_ops_rwsem);
1760         rtnl_lock();
1761         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1762         if (err)
1763                 goto unlock;
1764
1765         for_each_net(net)
1766                 call_netdevice_unregister_net_notifiers(nb, net);
1767
1768 unlock:
1769         rtnl_unlock();
1770         up_write(&pernet_ops_rwsem);
1771         return err;
1772 }
1773 EXPORT_SYMBOL(unregister_netdevice_notifier);
1774
1775 static int __register_netdevice_notifier_net(struct net *net,
1776                                              struct notifier_block *nb,
1777                                              bool ignore_call_fail)
1778 {
1779         int err;
1780
1781         err = raw_notifier_chain_register(&net->netdev_chain, nb);
1782         if (err)
1783                 return err;
1784         if (dev_boot_phase)
1785                 return 0;
1786
1787         err = call_netdevice_register_net_notifiers(nb, net);
1788         if (err && !ignore_call_fail)
1789                 goto chain_unregister;
1790
1791         return 0;
1792
1793 chain_unregister:
1794         raw_notifier_chain_unregister(&net->netdev_chain, nb);
1795         return err;
1796 }
1797
1798 static int __unregister_netdevice_notifier_net(struct net *net,
1799                                                struct notifier_block *nb)
1800 {
1801         int err;
1802
1803         err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1804         if (err)
1805                 return err;
1806
1807         call_netdevice_unregister_net_notifiers(nb, net);
1808         return 0;
1809 }
1810
1811 /**
1812  * register_netdevice_notifier_net - register a per-netns network notifier block
1813  * @net: network namespace
1814  * @nb: notifier
1815  *
1816  * Register a notifier to be called when network device events occur.
1817  * The notifier passed is linked into the kernel structures and must
1818  * not be reused until it has been unregistered. A negative errno code
1819  * is returned on a failure.
1820  *
1821  * When registered all registration and up events are replayed
1822  * to the new notifier to allow device to have a race free
1823  * view of the network device list.
1824  */
1825
1826 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1827 {
1828         int err;
1829
1830         rtnl_lock();
1831         err = __register_netdevice_notifier_net(net, nb, false);
1832         rtnl_unlock();
1833         return err;
1834 }
1835 EXPORT_SYMBOL(register_netdevice_notifier_net);
1836
1837 /**
1838  * unregister_netdevice_notifier_net - unregister a per-netns
1839  *                                     network notifier block
1840  * @net: network namespace
1841  * @nb: notifier
1842  *
1843  * Unregister a notifier previously registered by
1844  * register_netdevice_notifier_net(). The notifier is unlinked from the
1845  * kernel structures and may then be reused. A negative errno code
1846  * is returned on a failure.
1847  *
1848  * After unregistering unregister and down device events are synthesized
1849  * for all devices on the device list to the removed notifier to remove
1850  * the need for special case cleanup code.
1851  */
1852
1853 int unregister_netdevice_notifier_net(struct net *net,
1854                                       struct notifier_block *nb)
1855 {
1856         int err;
1857
1858         rtnl_lock();
1859         err = __unregister_netdevice_notifier_net(net, nb);
1860         rtnl_unlock();
1861         return err;
1862 }
1863 EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1864
1865 static void __move_netdevice_notifier_net(struct net *src_net,
1866                                           struct net *dst_net,
1867                                           struct notifier_block *nb)
1868 {
1869         __unregister_netdevice_notifier_net(src_net, nb);
1870         __register_netdevice_notifier_net(dst_net, nb, true);
1871 }
1872
1873 int register_netdevice_notifier_dev_net(struct net_device *dev,
1874                                         struct notifier_block *nb,
1875                                         struct netdev_net_notifier *nn)
1876 {
1877         int err;
1878
1879         rtnl_lock();
1880         err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
1881         if (!err) {
1882                 nn->nb = nb;
1883                 list_add(&nn->list, &dev->net_notifier_list);
1884         }
1885         rtnl_unlock();
1886         return err;
1887 }
1888 EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
1889
1890 int unregister_netdevice_notifier_dev_net(struct net_device *dev,
1891                                           struct notifier_block *nb,
1892                                           struct netdev_net_notifier *nn)
1893 {
1894         int err;
1895
1896         rtnl_lock();
1897         list_del(&nn->list);
1898         err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
1899         rtnl_unlock();
1900         return err;
1901 }
1902 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
1903
1904 static void move_netdevice_notifiers_dev_net(struct net_device *dev,
1905                                              struct net *net)
1906 {
1907         struct netdev_net_notifier *nn;
1908
1909         list_for_each_entry(nn, &dev->net_notifier_list, list)
1910                 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb);
1911 }
1912
1913 /**
1914  *      call_netdevice_notifiers_info - call all network notifier blocks
1915  *      @val: value passed unmodified to notifier function
1916  *      @info: notifier information data
1917  *
1918  *      Call all network notifier blocks.  Parameters and return value
1919  *      are as for raw_notifier_call_chain().
1920  */
1921
1922 static int call_netdevice_notifiers_info(unsigned long val,
1923                                          struct netdev_notifier_info *info)
1924 {
1925         struct net *net = dev_net(info->dev);
1926         int ret;
1927
1928         ASSERT_RTNL();
1929
1930         /* Run per-netns notifier block chain first, then run the global one.
1931          * Hopefully, one day, the global one is going to be removed after
1932          * all notifier block registrators get converted to be per-netns.
1933          */
1934         ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
1935         if (ret & NOTIFY_STOP_MASK)
1936                 return ret;
1937         return raw_notifier_call_chain(&netdev_chain, val, info);
1938 }
1939
1940 /**
1941  *      call_netdevice_notifiers_info_robust - call per-netns notifier blocks
1942  *                                             for and rollback on error
1943  *      @val_up: value passed unmodified to notifier function
1944  *      @val_down: value passed unmodified to the notifier function when
1945  *                 recovering from an error on @val_up
1946  *      @info: notifier information data
1947  *
1948  *      Call all per-netns network notifier blocks, but not notifier blocks on
1949  *      the global notifier chain. Parameters and return value are as for
1950  *      raw_notifier_call_chain_robust().
1951  */
1952
1953 static int
1954 call_netdevice_notifiers_info_robust(unsigned long val_up,
1955                                      unsigned long val_down,
1956                                      struct netdev_notifier_info *info)
1957 {
1958         struct net *net = dev_net(info->dev);
1959
1960         ASSERT_RTNL();
1961
1962         return raw_notifier_call_chain_robust(&net->netdev_chain,
1963                                               val_up, val_down, info);
1964 }
1965
1966 static int call_netdevice_notifiers_extack(unsigned long val,
1967                                            struct net_device *dev,
1968                                            struct netlink_ext_ack *extack)
1969 {
1970         struct netdev_notifier_info info = {
1971                 .dev = dev,
1972                 .extack = extack,
1973         };
1974
1975         return call_netdevice_notifiers_info(val, &info);
1976 }
1977
1978 /**
1979  *      call_netdevice_notifiers - call all network notifier blocks
1980  *      @val: value passed unmodified to notifier function
1981  *      @dev: net_device pointer passed unmodified to notifier function
1982  *
1983  *      Call all network notifier blocks.  Parameters and return value
1984  *      are as for raw_notifier_call_chain().
1985  */
1986
1987 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1988 {
1989         return call_netdevice_notifiers_extack(val, dev, NULL);
1990 }
1991 EXPORT_SYMBOL(call_netdevice_notifiers);
1992
1993 /**
1994  *      call_netdevice_notifiers_mtu - call all network notifier blocks
1995  *      @val: value passed unmodified to notifier function
1996  *      @dev: net_device pointer passed unmodified to notifier function
1997  *      @arg: additional u32 argument passed to the notifier function
1998  *
1999  *      Call all network notifier blocks.  Parameters and return value
2000  *      are as for raw_notifier_call_chain().
2001  */
2002 static int call_netdevice_notifiers_mtu(unsigned long val,
2003                                         struct net_device *dev, u32 arg)
2004 {
2005         struct netdev_notifier_info_ext info = {
2006                 .info.dev = dev,
2007                 .ext.mtu = arg,
2008         };
2009
2010         BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2011
2012         return call_netdevice_notifiers_info(val, &info.info);
2013 }
2014
2015 #ifdef CONFIG_NET_INGRESS
2016 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2017
2018 void net_inc_ingress_queue(void)
2019 {
2020         static_branch_inc(&ingress_needed_key);
2021 }
2022 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2023
2024 void net_dec_ingress_queue(void)
2025 {
2026         static_branch_dec(&ingress_needed_key);
2027 }
2028 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2029 #endif
2030
2031 #ifdef CONFIG_NET_EGRESS
2032 static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2033
2034 void net_inc_egress_queue(void)
2035 {
2036         static_branch_inc(&egress_needed_key);
2037 }
2038 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2039
2040 void net_dec_egress_queue(void)
2041 {
2042         static_branch_dec(&egress_needed_key);
2043 }
2044 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2045 #endif
2046
2047 DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2048 EXPORT_SYMBOL(netstamp_needed_key);
2049 #ifdef CONFIG_JUMP_LABEL
2050 static atomic_t netstamp_needed_deferred;
2051 static atomic_t netstamp_wanted;
2052 static void netstamp_clear(struct work_struct *work)
2053 {
2054         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2055         int wanted;
2056
2057         wanted = atomic_add_return(deferred, &netstamp_wanted);
2058         if (wanted > 0)
2059                 static_branch_enable(&netstamp_needed_key);
2060         else
2061                 static_branch_disable(&netstamp_needed_key);
2062 }
2063 static DECLARE_WORK(netstamp_work, netstamp_clear);
2064 #endif
2065
2066 void net_enable_timestamp(void)
2067 {
2068 #ifdef CONFIG_JUMP_LABEL
2069         int wanted = atomic_read(&netstamp_wanted);
2070
2071         while (wanted > 0) {
2072                 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1))
2073                         return;
2074         }
2075         atomic_inc(&netstamp_needed_deferred);
2076         schedule_work(&netstamp_work);
2077 #else
2078         static_branch_inc(&netstamp_needed_key);
2079 #endif
2080 }
2081 EXPORT_SYMBOL(net_enable_timestamp);
2082
2083 void net_disable_timestamp(void)
2084 {
2085 #ifdef CONFIG_JUMP_LABEL
2086         int wanted = atomic_read(&netstamp_wanted);
2087
2088         while (wanted > 1) {
2089                 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1))
2090                         return;
2091         }
2092         atomic_dec(&netstamp_needed_deferred);
2093         schedule_work(&netstamp_work);
2094 #else
2095         static_branch_dec(&netstamp_needed_key);
2096 #endif
2097 }
2098 EXPORT_SYMBOL(net_disable_timestamp);
2099
2100 static inline void net_timestamp_set(struct sk_buff *skb)
2101 {
2102         skb->tstamp = 0;
2103         skb->mono_delivery_time = 0;
2104         if (static_branch_unlikely(&netstamp_needed_key))
2105                 skb->tstamp = ktime_get_real();
2106 }
2107
2108 #define net_timestamp_check(COND, SKB)                          \
2109         if (static_branch_unlikely(&netstamp_needed_key)) {     \
2110                 if ((COND) && !(SKB)->tstamp)                   \
2111                         (SKB)->tstamp = ktime_get_real();       \
2112         }                                                       \
2113
2114 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2115 {
2116         return __is_skb_forwardable(dev, skb, true);
2117 }
2118 EXPORT_SYMBOL_GPL(is_skb_forwardable);
2119
2120 static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2121                               bool check_mtu)
2122 {
2123         int ret = ____dev_forward_skb(dev, skb, check_mtu);
2124
2125         if (likely(!ret)) {
2126                 skb->protocol = eth_type_trans(skb, dev);
2127                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2128         }
2129
2130         return ret;
2131 }
2132
2133 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2134 {
2135         return __dev_forward_skb2(dev, skb, true);
2136 }
2137 EXPORT_SYMBOL_GPL(__dev_forward_skb);
2138
2139 /**
2140  * dev_forward_skb - loopback an skb to another netif
2141  *
2142  * @dev: destination network device
2143  * @skb: buffer to forward
2144  *
2145  * return values:
2146  *      NET_RX_SUCCESS  (no congestion)
2147  *      NET_RX_DROP     (packet was dropped, but freed)
2148  *
2149  * dev_forward_skb can be used for injecting an skb from the
2150  * start_xmit function of one device into the receive queue
2151  * of another device.
2152  *
2153  * The receiving device may be in another namespace, so
2154  * we have to clear all information in the skb that could
2155  * impact namespace isolation.
2156  */
2157 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2158 {
2159         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2160 }
2161 EXPORT_SYMBOL_GPL(dev_forward_skb);
2162
2163 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2164 {
2165         return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2166 }
2167
2168 static inline int deliver_skb(struct sk_buff *skb,
2169                               struct packet_type *pt_prev,
2170                               struct net_device *orig_dev)
2171 {
2172         if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2173                 return -ENOMEM;
2174         refcount_inc(&skb->users);
2175         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2176 }
2177
2178 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2179                                           struct packet_type **pt,
2180                                           struct net_device *orig_dev,
2181                                           __be16 type,
2182                                           struct list_head *ptype_list)
2183 {
2184         struct packet_type *ptype, *pt_prev = *pt;
2185
2186         list_for_each_entry_rcu(ptype, ptype_list, list) {
2187                 if (ptype->type != type)
2188                         continue;
2189                 if (pt_prev)
2190                         deliver_skb(skb, pt_prev, orig_dev);
2191                 pt_prev = ptype;
2192         }
2193         *pt = pt_prev;
2194 }
2195
2196 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2197 {
2198         if (!ptype->af_packet_priv || !skb->sk)
2199                 return false;
2200
2201         if (ptype->id_match)
2202                 return ptype->id_match(ptype, skb->sk);
2203         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2204                 return true;
2205
2206         return false;
2207 }
2208
2209 /**
2210  * dev_nit_active - return true if any network interface taps are in use
2211  *
2212  * @dev: network device to check for the presence of taps
2213  */
2214 bool dev_nit_active(struct net_device *dev)
2215 {
2216         return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2217 }
2218 EXPORT_SYMBOL_GPL(dev_nit_active);
2219
2220 /*
2221  *      Support routine. Sends outgoing frames to any network
2222  *      taps currently in use.
2223  */
2224
2225 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2226 {
2227         struct packet_type *ptype;
2228         struct sk_buff *skb2 = NULL;
2229         struct packet_type *pt_prev = NULL;
2230         struct list_head *ptype_list = &ptype_all;
2231
2232         rcu_read_lock();
2233 again:
2234         list_for_each_entry_rcu(ptype, ptype_list, list) {
2235                 if (ptype->ignore_outgoing)
2236                         continue;
2237
2238                 /* Never send packets back to the socket
2239                  * they originated from - MvS (miquels@drinkel.ow.org)
2240                  */
2241                 if (skb_loop_sk(ptype, skb))
2242                         continue;
2243
2244                 if (pt_prev) {
2245                         deliver_skb(skb2, pt_prev, skb->dev);
2246                         pt_prev = ptype;
2247                         continue;
2248                 }
2249
2250                 /* need to clone skb, done only once */
2251                 skb2 = skb_clone(skb, GFP_ATOMIC);
2252                 if (!skb2)
2253                         goto out_unlock;
2254
2255                 net_timestamp_set(skb2);
2256
2257                 /* skb->nh should be correctly
2258                  * set by sender, so that the second statement is
2259                  * just protection against buggy protocols.
2260                  */
2261                 skb_reset_mac_header(skb2);
2262
2263                 if (skb_network_header(skb2) < skb2->data ||
2264                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2265                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2266                                              ntohs(skb2->protocol),
2267                                              dev->name);
2268                         skb_reset_network_header(skb2);
2269                 }
2270
2271                 skb2->transport_header = skb2->network_header;
2272                 skb2->pkt_type = PACKET_OUTGOING;
2273                 pt_prev = ptype;
2274         }
2275
2276         if (ptype_list == &ptype_all) {
2277                 ptype_list = &dev->ptype_all;
2278                 goto again;
2279         }
2280 out_unlock:
2281         if (pt_prev) {
2282                 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2283                         pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2284                 else
2285                         kfree_skb(skb2);
2286         }
2287         rcu_read_unlock();
2288 }
2289 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2290
2291 /**
2292  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2293  * @dev: Network device
2294  * @txq: number of queues available
2295  *
2296  * If real_num_tx_queues is changed the tc mappings may no longer be
2297  * valid. To resolve this verify the tc mapping remains valid and if
2298  * not NULL the mapping. With no priorities mapping to this
2299  * offset/count pair it will no longer be used. In the worst case TC0
2300  * is invalid nothing can be done so disable priority mappings. If is
2301  * expected that drivers will fix this mapping if they can before
2302  * calling netif_set_real_num_tx_queues.
2303  */
2304 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2305 {
2306         int i;
2307         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2308
2309         /* If TC0 is invalidated disable TC mapping */
2310         if (tc->offset + tc->count > txq) {
2311                 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2312                 dev->num_tc = 0;
2313                 return;
2314         }
2315
2316         /* Invalidated prio to tc mappings set to TC0 */
2317         for (i = 1; i < TC_BITMASK + 1; i++) {
2318                 int q = netdev_get_prio_tc_map(dev, i);
2319
2320                 tc = &dev->tc_to_txq[q];
2321                 if (tc->offset + tc->count > txq) {
2322                         netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2323                                     i, q);
2324                         netdev_set_prio_tc_map(dev, i, 0);
2325                 }
2326         }
2327 }
2328
2329 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2330 {
2331         if (dev->num_tc) {
2332                 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2333                 int i;
2334
2335                 /* walk through the TCs and see if it falls into any of them */
2336                 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2337                         if ((txq - tc->offset) < tc->count)
2338                                 return i;
2339                 }
2340
2341                 /* didn't find it, just return -1 to indicate no match */
2342                 return -1;
2343         }
2344
2345         return 0;
2346 }
2347 EXPORT_SYMBOL(netdev_txq_to_tc);
2348
2349 #ifdef CONFIG_XPS
2350 static struct static_key xps_needed __read_mostly;
2351 static struct static_key xps_rxqs_needed __read_mostly;
2352 static DEFINE_MUTEX(xps_map_mutex);
2353 #define xmap_dereference(P)             \
2354         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2355
2356 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2357                              struct xps_dev_maps *old_maps, int tci, u16 index)
2358 {
2359         struct xps_map *map = NULL;
2360         int pos;
2361
2362         if (dev_maps)
2363                 map = xmap_dereference(dev_maps->attr_map[tci]);
2364         if (!map)
2365                 return false;
2366
2367         for (pos = map->len; pos--;) {
2368                 if (map->queues[pos] != index)
2369                         continue;
2370
2371                 if (map->len > 1) {
2372                         map->queues[pos] = map->queues[--map->len];
2373                         break;
2374                 }
2375
2376                 if (old_maps)
2377                         RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2378                 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2379                 kfree_rcu(map, rcu);
2380                 return false;
2381         }
2382
2383         return true;
2384 }
2385
2386 static bool remove_xps_queue_cpu(struct net_device *dev,
2387                                  struct xps_dev_maps *dev_maps,
2388                                  int cpu, u16 offset, u16 count)
2389 {
2390         int num_tc = dev_maps->num_tc;
2391         bool active = false;
2392         int tci;
2393
2394         for (tci = cpu * num_tc; num_tc--; tci++) {
2395                 int i, j;
2396
2397                 for (i = count, j = offset; i--; j++) {
2398                         if (!remove_xps_queue(dev_maps, NULL, tci, j))
2399                                 break;
2400                 }
2401
2402                 active |= i < 0;
2403         }
2404
2405         return active;
2406 }
2407
2408 static void reset_xps_maps(struct net_device *dev,
2409                            struct xps_dev_maps *dev_maps,
2410                            enum xps_map_type type)
2411 {
2412         static_key_slow_dec_cpuslocked(&xps_needed);
2413         if (type == XPS_RXQS)
2414                 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2415
2416         RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2417
2418         kfree_rcu(dev_maps, rcu);
2419 }
2420
2421 static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2422                            u16 offset, u16 count)
2423 {
2424         struct xps_dev_maps *dev_maps;
2425         bool active = false;
2426         int i, j;
2427
2428         dev_maps = xmap_dereference(dev->xps_maps[type]);
2429         if (!dev_maps)
2430                 return;
2431
2432         for (j = 0; j < dev_maps->nr_ids; j++)
2433                 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2434         if (!active)
2435                 reset_xps_maps(dev, dev_maps, type);
2436
2437         if (type == XPS_CPUS) {
2438                 for (i = offset + (count - 1); count--; i--)
2439                         netdev_queue_numa_node_write(
2440                                 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2441         }
2442 }
2443
2444 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2445                                    u16 count)
2446 {
2447         if (!static_key_false(&xps_needed))
2448                 return;
2449
2450         cpus_read_lock();
2451         mutex_lock(&xps_map_mutex);
2452
2453         if (static_key_false(&xps_rxqs_needed))
2454                 clean_xps_maps(dev, XPS_RXQS, offset, count);
2455
2456         clean_xps_maps(dev, XPS_CPUS, offset, count);
2457
2458         mutex_unlock(&xps_map_mutex);
2459         cpus_read_unlock();
2460 }
2461
2462 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2463 {
2464         netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2465 }
2466
2467 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2468                                       u16 index, bool is_rxqs_map)
2469 {
2470         struct xps_map *new_map;
2471         int alloc_len = XPS_MIN_MAP_ALLOC;
2472         int i, pos;
2473
2474         for (pos = 0; map && pos < map->len; pos++) {
2475                 if (map->queues[pos] != index)
2476                         continue;
2477                 return map;
2478         }
2479
2480         /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2481         if (map) {
2482                 if (pos < map->alloc_len)
2483                         return map;
2484
2485                 alloc_len = map->alloc_len * 2;
2486         }
2487
2488         /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2489          *  map
2490          */
2491         if (is_rxqs_map)
2492                 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2493         else
2494                 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2495                                        cpu_to_node(attr_index));
2496         if (!new_map)
2497                 return NULL;
2498
2499         for (i = 0; i < pos; i++)
2500                 new_map->queues[i] = map->queues[i];
2501         new_map->alloc_len = alloc_len;
2502         new_map->len = pos;
2503
2504         return new_map;
2505 }
2506
2507 /* Copy xps maps at a given index */
2508 static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2509                               struct xps_dev_maps *new_dev_maps, int index,
2510                               int tc, bool skip_tc)
2511 {
2512         int i, tci = index * dev_maps->num_tc;
2513         struct xps_map *map;
2514
2515         /* copy maps belonging to foreign traffic classes */
2516         for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2517                 if (i == tc && skip_tc)
2518                         continue;
2519
2520                 /* fill in the new device map from the old device map */
2521                 map = xmap_dereference(dev_maps->attr_map[tci]);
2522                 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2523         }
2524 }
2525
2526 /* Must be called under cpus_read_lock */
2527 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2528                           u16 index, enum xps_map_type type)
2529 {
2530         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2531         const unsigned long *online_mask = NULL;
2532         bool active = false, copy = false;
2533         int i, j, tci, numa_node_id = -2;
2534         int maps_sz, num_tc = 1, tc = 0;
2535         struct xps_map *map, *new_map;
2536         unsigned int nr_ids;
2537
2538         WARN_ON_ONCE(index >= dev->num_tx_queues);
2539
2540         if (dev->num_tc) {
2541                 /* Do not allow XPS on subordinate device directly */
2542                 num_tc = dev->num_tc;
2543                 if (num_tc < 0)
2544                         return -EINVAL;
2545
2546                 /* If queue belongs to subordinate dev use its map */
2547                 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2548
2549                 tc = netdev_txq_to_tc(dev, index);
2550                 if (tc < 0)
2551                         return -EINVAL;
2552         }
2553
2554         mutex_lock(&xps_map_mutex);
2555
2556         dev_maps = xmap_dereference(dev->xps_maps[type]);
2557         if (type == XPS_RXQS) {
2558                 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2559                 nr_ids = dev->num_rx_queues;
2560         } else {
2561                 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2562                 if (num_possible_cpus() > 1)
2563                         online_mask = cpumask_bits(cpu_online_mask);
2564                 nr_ids = nr_cpu_ids;
2565         }
2566
2567         if (maps_sz < L1_CACHE_BYTES)
2568                 maps_sz = L1_CACHE_BYTES;
2569
2570         /* The old dev_maps could be larger or smaller than the one we're
2571          * setting up now, as dev->num_tc or nr_ids could have been updated in
2572          * between. We could try to be smart, but let's be safe instead and only
2573          * copy foreign traffic classes if the two map sizes match.
2574          */
2575         if (dev_maps &&
2576             dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2577                 copy = true;
2578
2579         /* allocate memory for queue storage */
2580         for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2581              j < nr_ids;) {
2582                 if (!new_dev_maps) {
2583                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2584                         if (!new_dev_maps) {
2585                                 mutex_unlock(&xps_map_mutex);
2586                                 return -ENOMEM;
2587                         }
2588
2589                         new_dev_maps->nr_ids = nr_ids;
2590                         new_dev_maps->num_tc = num_tc;
2591                 }
2592
2593                 tci = j * num_tc + tc;
2594                 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2595
2596                 map = expand_xps_map(map, j, index, type == XPS_RXQS);
2597                 if (!map)
2598                         goto error;
2599
2600                 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2601         }
2602
2603         if (!new_dev_maps)
2604                 goto out_no_new_maps;
2605
2606         if (!dev_maps) {
2607                 /* Increment static keys at most once per type */
2608                 static_key_slow_inc_cpuslocked(&xps_needed);
2609                 if (type == XPS_RXQS)
2610                         static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2611         }
2612
2613         for (j = 0; j < nr_ids; j++) {
2614                 bool skip_tc = false;
2615
2616                 tci = j * num_tc + tc;
2617                 if (netif_attr_test_mask(j, mask, nr_ids) &&
2618                     netif_attr_test_online(j, online_mask, nr_ids)) {
2619                         /* add tx-queue to CPU/rx-queue maps */
2620                         int pos = 0;
2621
2622                         skip_tc = true;
2623
2624                         map = xmap_dereference(new_dev_maps->attr_map[tci]);
2625                         while ((pos < map->len) && (map->queues[pos] != index))
2626                                 pos++;
2627
2628                         if (pos == map->len)
2629                                 map->queues[map->len++] = index;
2630 #ifdef CONFIG_NUMA
2631                         if (type == XPS_CPUS) {
2632                                 if (numa_node_id == -2)
2633                                         numa_node_id = cpu_to_node(j);
2634                                 else if (numa_node_id != cpu_to_node(j))
2635                                         numa_node_id = -1;
2636                         }
2637 #endif
2638                 }
2639
2640                 if (copy)
2641                         xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2642                                           skip_tc);
2643         }
2644
2645         rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2646
2647         /* Cleanup old maps */
2648         if (!dev_maps)
2649                 goto out_no_old_maps;
2650
2651         for (j = 0; j < dev_maps->nr_ids; j++) {
2652                 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2653                         map = xmap_dereference(dev_maps->attr_map[tci]);
2654                         if (!map)
2655                                 continue;
2656
2657                         if (copy) {
2658                                 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2659                                 if (map == new_map)
2660                                         continue;
2661                         }
2662
2663                         RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2664                         kfree_rcu(map, rcu);
2665                 }
2666         }
2667
2668         old_dev_maps = dev_maps;
2669
2670 out_no_old_maps:
2671         dev_maps = new_dev_maps;
2672         active = true;
2673
2674 out_no_new_maps:
2675         if (type == XPS_CPUS)
2676                 /* update Tx queue numa node */
2677                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2678                                              (numa_node_id >= 0) ?
2679                                              numa_node_id : NUMA_NO_NODE);
2680
2681         if (!dev_maps)
2682                 goto out_no_maps;
2683
2684         /* removes tx-queue from unused CPUs/rx-queues */
2685         for (j = 0; j < dev_maps->nr_ids; j++) {
2686                 tci = j * dev_maps->num_tc;
2687
2688                 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2689                         if (i == tc &&
2690                             netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2691                             netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2692                                 continue;
2693
2694                         active |= remove_xps_queue(dev_maps,
2695                                                    copy ? old_dev_maps : NULL,
2696                                                    tci, index);
2697                 }
2698         }
2699
2700         if (old_dev_maps)
2701                 kfree_rcu(old_dev_maps, rcu);
2702
2703         /* free map if not active */
2704         if (!active)
2705                 reset_xps_maps(dev, dev_maps, type);
2706
2707 out_no_maps:
2708         mutex_unlock(&xps_map_mutex);
2709
2710         return 0;
2711 error:
2712         /* remove any maps that we added */
2713         for (j = 0; j < nr_ids; j++) {
2714                 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2715                         new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2716                         map = copy ?
2717                               xmap_dereference(dev_maps->attr_map[tci]) :
2718                               NULL;
2719                         if (new_map && new_map != map)
2720                                 kfree(new_map);
2721                 }
2722         }
2723
2724         mutex_unlock(&xps_map_mutex);
2725
2726         kfree(new_dev_maps);
2727         return -ENOMEM;
2728 }
2729 EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2730
2731 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2732                         u16 index)
2733 {
2734         int ret;
2735
2736         cpus_read_lock();
2737         ret =  __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2738         cpus_read_unlock();
2739
2740         return ret;
2741 }
2742 EXPORT_SYMBOL(netif_set_xps_queue);
2743
2744 #endif
2745 static void netdev_unbind_all_sb_channels(struct net_device *dev)
2746 {
2747         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2748
2749         /* Unbind any subordinate channels */
2750         while (txq-- != &dev->_tx[0]) {
2751                 if (txq->sb_dev)
2752                         netdev_unbind_sb_channel(dev, txq->sb_dev);
2753         }
2754 }
2755
2756 void netdev_reset_tc(struct net_device *dev)
2757 {
2758 #ifdef CONFIG_XPS
2759         netif_reset_xps_queues_gt(dev, 0);
2760 #endif
2761         netdev_unbind_all_sb_channels(dev);
2762
2763         /* Reset TC configuration of device */
2764         dev->num_tc = 0;
2765         memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2766         memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2767 }
2768 EXPORT_SYMBOL(netdev_reset_tc);
2769
2770 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2771 {
2772         if (tc >= dev->num_tc)
2773                 return -EINVAL;
2774
2775 #ifdef CONFIG_XPS
2776         netif_reset_xps_queues(dev, offset, count);
2777 #endif
2778         dev->tc_to_txq[tc].count = count;
2779         dev->tc_to_txq[tc].offset = offset;
2780         return 0;
2781 }
2782 EXPORT_SYMBOL(netdev_set_tc_queue);
2783
2784 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2785 {
2786         if (num_tc > TC_MAX_QUEUE)
2787                 return -EINVAL;
2788
2789 #ifdef CONFIG_XPS
2790         netif_reset_xps_queues_gt(dev, 0);
2791 #endif
2792         netdev_unbind_all_sb_channels(dev);
2793
2794         dev->num_tc = num_tc;
2795         return 0;
2796 }
2797 EXPORT_SYMBOL(netdev_set_num_tc);
2798
2799 void netdev_unbind_sb_channel(struct net_device *dev,
2800                               struct net_device *sb_dev)
2801 {
2802         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2803
2804 #ifdef CONFIG_XPS
2805         netif_reset_xps_queues_gt(sb_dev, 0);
2806 #endif
2807         memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2808         memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2809
2810         while (txq-- != &dev->_tx[0]) {
2811                 if (txq->sb_dev == sb_dev)
2812                         txq->sb_dev = NULL;
2813         }
2814 }
2815 EXPORT_SYMBOL(netdev_unbind_sb_channel);
2816
2817 int netdev_bind_sb_channel_queue(struct net_device *dev,
2818                                  struct net_device *sb_dev,
2819                                  u8 tc, u16 count, u16 offset)
2820 {
2821         /* Make certain the sb_dev and dev are already configured */
2822         if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2823                 return -EINVAL;
2824
2825         /* We cannot hand out queues we don't have */
2826         if ((offset + count) > dev->real_num_tx_queues)
2827                 return -EINVAL;
2828
2829         /* Record the mapping */
2830         sb_dev->tc_to_txq[tc].count = count;
2831         sb_dev->tc_to_txq[tc].offset = offset;
2832
2833         /* Provide a way for Tx queue to find the tc_to_txq map or
2834          * XPS map for itself.
2835          */
2836         while (count--)
2837                 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2838
2839         return 0;
2840 }
2841 EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2842
2843 int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2844 {
2845         /* Do not use a multiqueue device to represent a subordinate channel */
2846         if (netif_is_multiqueue(dev))
2847                 return -ENODEV;
2848
2849         /* We allow channels 1 - 32767 to be used for subordinate channels.
2850          * Channel 0 is meant to be "native" mode and used only to represent
2851          * the main root device. We allow writing 0 to reset the device back
2852          * to normal mode after being used as a subordinate channel.
2853          */
2854         if (channel > S16_MAX)
2855                 return -EINVAL;
2856
2857         dev->num_tc = -channel;
2858
2859         return 0;
2860 }
2861 EXPORT_SYMBOL(netdev_set_sb_channel);
2862
2863 /*
2864  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2865  * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2866  */
2867 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2868 {
2869         bool disabling;
2870         int rc;
2871
2872         disabling = txq < dev->real_num_tx_queues;
2873
2874         if (txq < 1 || txq > dev->num_tx_queues)
2875                 return -EINVAL;
2876
2877         if (dev->reg_state == NETREG_REGISTERED ||
2878             dev->reg_state == NETREG_UNREGISTERING) {
2879                 ASSERT_RTNL();
2880
2881                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2882                                                   txq);
2883                 if (rc)
2884                         return rc;
2885
2886                 if (dev->num_tc)
2887                         netif_setup_tc(dev, txq);
2888
2889                 dev_qdisc_change_real_num_tx(dev, txq);
2890
2891                 dev->real_num_tx_queues = txq;
2892
2893                 if (disabling) {
2894                         synchronize_net();
2895                         qdisc_reset_all_tx_gt(dev, txq);
2896 #ifdef CONFIG_XPS
2897                         netif_reset_xps_queues_gt(dev, txq);
2898 #endif
2899                 }
2900         } else {
2901                 dev->real_num_tx_queues = txq;
2902         }
2903
2904         return 0;
2905 }
2906 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2907
2908 #ifdef CONFIG_SYSFS
2909 /**
2910  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2911  *      @dev: Network device
2912  *      @rxq: Actual number of RX queues
2913  *
2914  *      This must be called either with the rtnl_lock held or before
2915  *      registration of the net device.  Returns 0 on success, or a
2916  *      negative error code.  If called before registration, it always
2917  *      succeeds.
2918  */
2919 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2920 {
2921         int rc;
2922
2923         if (rxq < 1 || rxq > dev->num_rx_queues)
2924                 return -EINVAL;
2925
2926         if (dev->reg_state == NETREG_REGISTERED) {
2927                 ASSERT_RTNL();
2928
2929                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2930                                                   rxq);
2931                 if (rc)
2932                         return rc;
2933         }
2934
2935         dev->real_num_rx_queues = rxq;
2936         return 0;
2937 }
2938 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2939 #endif
2940
2941 /**
2942  *      netif_set_real_num_queues - set actual number of RX and TX queues used
2943  *      @dev: Network device
2944  *      @txq: Actual number of TX queues
2945  *      @rxq: Actual number of RX queues
2946  *
2947  *      Set the real number of both TX and RX queues.
2948  *      Does nothing if the number of queues is already correct.
2949  */
2950 int netif_set_real_num_queues(struct net_device *dev,
2951                               unsigned int txq, unsigned int rxq)
2952 {
2953         unsigned int old_rxq = dev->real_num_rx_queues;
2954         int err;
2955
2956         if (txq < 1 || txq > dev->num_tx_queues ||
2957             rxq < 1 || rxq > dev->num_rx_queues)
2958                 return -EINVAL;
2959
2960         /* Start from increases, so the error path only does decreases -
2961          * decreases can't fail.
2962          */
2963         if (rxq > dev->real_num_rx_queues) {
2964                 err = netif_set_real_num_rx_queues(dev, rxq);
2965                 if (err)
2966                         return err;
2967         }
2968         if (txq > dev->real_num_tx_queues) {
2969                 err = netif_set_real_num_tx_queues(dev, txq);
2970                 if (err)
2971                         goto undo_rx;
2972         }
2973         if (rxq < dev->real_num_rx_queues)
2974                 WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
2975         if (txq < dev->real_num_tx_queues)
2976                 WARN_ON(netif_set_real_num_tx_queues(dev, txq));
2977
2978         return 0;
2979 undo_rx:
2980         WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
2981         return err;
2982 }
2983 EXPORT_SYMBOL(netif_set_real_num_queues);
2984
2985 /**
2986  * netif_set_tso_max_size() - set the max size of TSO frames supported
2987  * @dev:        netdev to update
2988  * @size:       max skb->len of a TSO frame
2989  *
2990  * Set the limit on the size of TSO super-frames the device can handle.
2991  * Unless explicitly set the stack will assume the value of
2992  * %GSO_LEGACY_MAX_SIZE.
2993  */
2994 void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
2995 {
2996         dev->tso_max_size = min(GSO_MAX_SIZE, size);
2997         if (size < READ_ONCE(dev->gso_max_size))
2998                 netif_set_gso_max_size(dev, size);
2999         if (size < READ_ONCE(dev->gso_ipv4_max_size))
3000                 netif_set_gso_ipv4_max_size(dev, size);
3001 }
3002 EXPORT_SYMBOL(netif_set_tso_max_size);
3003
3004 /**
3005  * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3006  * @dev:        netdev to update
3007  * @segs:       max number of TCP segments
3008  *
3009  * Set the limit on the number of TCP segments the device can generate from
3010  * a single TSO super-frame.
3011  * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3012  */
3013 void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3014 {
3015         dev->tso_max_segs = segs;
3016         if (segs < READ_ONCE(dev->gso_max_segs))
3017                 netif_set_gso_max_segs(dev, segs);
3018 }
3019 EXPORT_SYMBOL(netif_set_tso_max_segs);
3020
3021 /**
3022  * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3023  * @to:         netdev to update
3024  * @from:       netdev from which to copy the limits
3025  */
3026 void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
3027 {
3028         netif_set_tso_max_size(to, from->tso_max_size);
3029         netif_set_tso_max_segs(to, from->tso_max_segs);
3030 }
3031 EXPORT_SYMBOL(netif_inherit_tso_max);
3032
3033 /**
3034  * netif_get_num_default_rss_queues - default number of RSS queues
3035  *
3036  * Default value is the number of physical cores if there are only 1 or 2, or
3037  * divided by 2 if there are more.
3038  */
3039 int netif_get_num_default_rss_queues(void)
3040 {
3041         cpumask_var_t cpus;
3042         int cpu, count = 0;
3043
3044         if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3045                 return 1;
3046
3047         cpumask_copy(cpus, cpu_online_mask);
3048         for_each_cpu(cpu, cpus) {
3049                 ++count;
3050                 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3051         }
3052         free_cpumask_var(cpus);
3053
3054         return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3055 }
3056 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3057
3058 static void __netif_reschedule(struct Qdisc *q)
3059 {
3060         struct softnet_data *sd;
3061         unsigned long flags;
3062
3063         local_irq_save(flags);
3064         sd = this_cpu_ptr(&softnet_data);
3065         q->next_sched = NULL;
3066         *sd->output_queue_tailp = q;
3067         sd->output_queue_tailp = &q->next_sched;
3068         raise_softirq_irqoff(NET_TX_SOFTIRQ);
3069         local_irq_restore(flags);
3070 }
3071
3072 void __netif_schedule(struct Qdisc *q)
3073 {
3074         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3075                 __netif_reschedule(q);
3076 }
3077 EXPORT_SYMBOL(__netif_schedule);
3078
3079 struct dev_kfree_skb_cb {
3080         enum skb_drop_reason reason;
3081 };
3082
3083 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3084 {
3085         return (struct dev_kfree_skb_cb *)skb->cb;
3086 }
3087
3088 void netif_schedule_queue(struct netdev_queue *txq)
3089 {
3090         rcu_read_lock();
3091         if (!netif_xmit_stopped(txq)) {
3092                 struct Qdisc *q = rcu_dereference(txq->qdisc);
3093
3094                 __netif_schedule(q);
3095         }
3096         rcu_read_unlock();
3097 }
3098 EXPORT_SYMBOL(netif_schedule_queue);
3099
3100 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3101 {
3102         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3103                 struct Qdisc *q;
3104
3105                 rcu_read_lock();
3106                 q = rcu_dereference(dev_queue->qdisc);
3107                 __netif_schedule(q);
3108                 rcu_read_unlock();
3109         }
3110 }
3111 EXPORT_SYMBOL(netif_tx_wake_queue);
3112
3113 void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3114 {
3115         unsigned long flags;
3116
3117         if (unlikely(!skb))
3118                 return;
3119
3120         if (likely(refcount_read(&skb->users) == 1)) {
3121                 smp_rmb();
3122                 refcount_set(&skb->users, 0);
3123         } else if (likely(!refcount_dec_and_test(&skb->users))) {
3124                 return;
3125         }
3126         get_kfree_skb_cb(skb)->reason = reason;
3127         local_irq_save(flags);
3128         skb->next = __this_cpu_read(softnet_data.completion_queue);
3129         __this_cpu_write(softnet_data.completion_queue, skb);
3130         raise_softirq_irqoff(NET_TX_SOFTIRQ);
3131         local_irq_restore(flags);
3132 }
3133 EXPORT_SYMBOL(dev_kfree_skb_irq_reason);
3134
3135 void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3136 {
3137         if (in_hardirq() || irqs_disabled())
3138                 dev_kfree_skb_irq_reason(skb, reason);
3139         else
3140                 kfree_skb_reason(skb, reason);
3141 }
3142 EXPORT_SYMBOL(dev_kfree_skb_any_reason);
3143
3144
3145 /**
3146  * netif_device_detach - mark device as removed
3147  * @dev: network device
3148  *
3149  * Mark device as removed from system and therefore no longer available.
3150  */
3151 void netif_device_detach(struct net_device *dev)
3152 {
3153         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3154             netif_running(dev)) {
3155                 netif_tx_stop_all_queues(dev);
3156         }
3157 }
3158 EXPORT_SYMBOL(netif_device_detach);
3159
3160 /**
3161  * netif_device_attach - mark device as attached
3162  * @dev: network device
3163  *
3164  * Mark device as attached from system and restart if needed.
3165  */
3166 void netif_device_attach(struct net_device *dev)
3167 {
3168         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3169             netif_running(dev)) {
3170                 netif_tx_wake_all_queues(dev);
3171                 __netdev_watchdog_up(dev);
3172         }
3173 }
3174 EXPORT_SYMBOL(netif_device_attach);
3175
3176 /*
3177  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3178  * to be used as a distribution range.
3179  */
3180 static u16 skb_tx_hash(const struct net_device *dev,
3181                        const struct net_device *sb_dev,
3182                        struct sk_buff *skb)
3183 {
3184         u32 hash;
3185         u16 qoffset = 0;
3186         u16 qcount = dev->real_num_tx_queues;
3187
3188         if (dev->num_tc) {
3189                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3190
3191                 qoffset = sb_dev->tc_to_txq[tc].offset;
3192                 qcount = sb_dev->tc_to_txq[tc].count;
3193                 if (unlikely(!qcount)) {
3194                         net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3195                                              sb_dev->name, qoffset, tc);
3196                         qoffset = 0;
3197                         qcount = dev->real_num_tx_queues;
3198                 }
3199         }
3200
3201         if (skb_rx_queue_recorded(skb)) {
3202                 hash = skb_get_rx_queue(skb);
3203                 if (hash >= qoffset)
3204                         hash -= qoffset;
3205                 while (unlikely(hash >= qcount))
3206                         hash -= qcount;
3207                 return hash + qoffset;
3208         }
3209
3210         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3211 }
3212
3213 static void skb_warn_bad_offload(const struct sk_buff *skb)
3214 {
3215         static const netdev_features_t null_features;
3216         struct net_device *dev = skb->dev;
3217         const char *name = "";
3218
3219         if (!net_ratelimit())
3220                 return;
3221
3222         if (dev) {
3223                 if (dev->dev.parent)
3224                         name = dev_driver_string(dev->dev.parent);
3225                 else
3226                         name = netdev_name(dev);
3227         }
3228         skb_dump(KERN_WARNING, skb, false);
3229         WARN(1, "%s: caps=(%pNF, %pNF)\n",
3230              name, dev ? &dev->features : &null_features,
3231              skb->sk ? &skb->sk->sk_route_caps : &null_features);
3232 }
3233
3234 /*
3235  * Invalidate hardware checksum when packet is to be mangled, and
3236  * complete checksum manually on outgoing path.
3237  */
3238 int skb_checksum_help(struct sk_buff *skb)
3239 {
3240         __wsum csum;
3241         int ret = 0, offset;
3242
3243         if (skb->ip_summed == CHECKSUM_COMPLETE)
3244                 goto out_set_summed;
3245
3246         if (unlikely(skb_is_gso(skb))) {
3247                 skb_warn_bad_offload(skb);
3248                 return -EINVAL;
3249         }
3250
3251         /* Before computing a checksum, we should make sure no frag could
3252          * be modified by an external entity : checksum could be wrong.
3253          */
3254         if (skb_has_shared_frag(skb)) {
3255                 ret = __skb_linearize(skb);
3256                 if (ret)
3257                         goto out;
3258         }
3259
3260         offset = skb_checksum_start_offset(skb);
3261         ret = -EINVAL;
3262         if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3263                 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3264                 goto out;
3265         }
3266         csum = skb_checksum(skb, offset, skb->len - offset, 0);
3267
3268         offset += skb->csum_offset;
3269         if (WARN_ON_ONCE(offset + sizeof(__sum16) > skb_headlen(skb))) {
3270                 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3271                 goto out;
3272         }
3273         ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3274         if (ret)
3275                 goto out;
3276
3277         *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3278 out_set_summed:
3279         skb->ip_summed = CHECKSUM_NONE;
3280 out:
3281         return ret;
3282 }
3283 EXPORT_SYMBOL(skb_checksum_help);
3284
3285 int skb_crc32c_csum_help(struct sk_buff *skb)
3286 {
3287         __le32 crc32c_csum;
3288         int ret = 0, offset, start;
3289
3290         if (skb->ip_summed != CHECKSUM_PARTIAL)
3291                 goto out;
3292
3293         if (unlikely(skb_is_gso(skb)))
3294                 goto out;
3295
3296         /* Before computing a checksum, we should make sure no frag could
3297          * be modified by an external entity : checksum could be wrong.
3298          */
3299         if (unlikely(skb_has_shared_frag(skb))) {
3300                 ret = __skb_linearize(skb);
3301                 if (ret)
3302                         goto out;
3303         }
3304         start = skb_checksum_start_offset(skb);
3305         offset = start + offsetof(struct sctphdr, checksum);
3306         if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3307                 ret = -EINVAL;
3308                 goto out;
3309         }
3310
3311         ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3312         if (ret)
3313                 goto out;
3314
3315         crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3316                                                   skb->len - start, ~(__u32)0,
3317                                                   crc32c_csum_stub));
3318         *(__le32 *)(skb->data + offset) = crc32c_csum;
3319         skb->ip_summed = CHECKSUM_NONE;
3320         skb->csum_not_inet = 0;
3321 out:
3322         return ret;
3323 }
3324
3325 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3326 {
3327         __be16 type = skb->protocol;
3328
3329         /* Tunnel gso handlers can set protocol to ethernet. */
3330         if (type == htons(ETH_P_TEB)) {
3331                 struct ethhdr *eth;
3332
3333                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3334                         return 0;
3335
3336                 eth = (struct ethhdr *)skb->data;
3337                 type = eth->h_proto;
3338         }
3339
3340         return __vlan_get_protocol(skb, type, depth);
3341 }
3342
3343 /* openvswitch calls this on rx path, so we need a different check.
3344  */
3345 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3346 {
3347         if (tx_path)
3348                 return skb->ip_summed != CHECKSUM_PARTIAL &&
3349                        skb->ip_summed != CHECKSUM_UNNECESSARY;
3350
3351         return skb->ip_summed == CHECKSUM_NONE;
3352 }
3353
3354 /**
3355  *      __skb_gso_segment - Perform segmentation on skb.
3356  *      @skb: buffer to segment
3357  *      @features: features for the output path (see dev->features)
3358  *      @tx_path: whether it is called in TX path
3359  *
3360  *      This function segments the given skb and returns a list of segments.
3361  *
3362  *      It may return NULL if the skb requires no segmentation.  This is
3363  *      only possible when GSO is used for verifying header integrity.
3364  *
3365  *      Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
3366  */
3367 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3368                                   netdev_features_t features, bool tx_path)
3369 {
3370         struct sk_buff *segs;
3371
3372         if (unlikely(skb_needs_check(skb, tx_path))) {
3373                 int err;
3374
3375                 /* We're going to init ->check field in TCP or UDP header */
3376                 err = skb_cow_head(skb, 0);
3377                 if (err < 0)
3378                         return ERR_PTR(err);
3379         }
3380
3381         /* Only report GSO partial support if it will enable us to
3382          * support segmentation on this frame without needing additional
3383          * work.
3384          */
3385         if (features & NETIF_F_GSO_PARTIAL) {
3386                 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3387                 struct net_device *dev = skb->dev;
3388
3389                 partial_features |= dev->features & dev->gso_partial_features;
3390                 if (!skb_gso_ok(skb, features | partial_features))
3391                         features &= ~NETIF_F_GSO_PARTIAL;
3392         }
3393
3394         BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
3395                      sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3396
3397         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3398         SKB_GSO_CB(skb)->encap_level = 0;
3399
3400         skb_reset_mac_header(skb);
3401         skb_reset_mac_len(skb);
3402
3403         segs = skb_mac_gso_segment(skb, features);
3404
3405         if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3406                 skb_warn_bad_offload(skb);
3407
3408         return segs;
3409 }
3410 EXPORT_SYMBOL(__skb_gso_segment);
3411
3412 /* Take action when hardware reception checksum errors are detected. */
3413 #ifdef CONFIG_BUG
3414 static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3415 {
3416         netdev_err(dev, "hw csum failure\n");
3417         skb_dump(KERN_ERR, skb, true);
3418         dump_stack();
3419 }
3420
3421 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3422 {
3423         DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3424 }
3425 EXPORT_SYMBOL(netdev_rx_csum_fault);
3426 #endif
3427
3428 /* XXX: check that highmem exists at all on the given machine. */
3429 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3430 {
3431 #ifdef CONFIG_HIGHMEM
3432         int i;
3433
3434         if (!(dev->features & NETIF_F_HIGHDMA)) {
3435                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3436                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3437
3438                         if (PageHighMem(skb_frag_page(frag)))
3439                                 return 1;
3440                 }
3441         }
3442 #endif
3443         return 0;
3444 }
3445
3446 /* If MPLS offload request, verify we are testing hardware MPLS features
3447  * instead of standard features for the netdev.
3448  */
3449 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3450 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3451                                            netdev_features_t features,
3452                                            __be16 type)
3453 {
3454         if (eth_p_mpls(type))
3455                 features &= skb->dev->mpls_features;
3456
3457         return features;
3458 }
3459 #else
3460 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3461                                            netdev_features_t features,
3462                                            __be16 type)
3463 {
3464         return features;
3465 }
3466 #endif
3467
3468 static netdev_features_t harmonize_features(struct sk_buff *skb,
3469         netdev_features_t features)
3470 {
3471         __be16 type;
3472
3473         type = skb_network_protocol(skb, NULL);
3474         features = net_mpls_features(skb, features, type);
3475
3476         if (skb->ip_summed != CHECKSUM_NONE &&
3477             !can_checksum_protocol(features, type)) {
3478                 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3479         }
3480         if (illegal_highdma(skb->dev, skb))
3481                 features &= ~NETIF_F_SG;
3482
3483         return features;
3484 }
3485
3486 netdev_features_t passthru_features_check(struct sk_buff *skb,
3487                                           struct net_device *dev,
3488                                           netdev_features_t features)
3489 {
3490         return features;
3491 }
3492 EXPORT_SYMBOL(passthru_features_check);
3493
3494 static netdev_features_t dflt_features_check(struct sk_buff *skb,
3495                                              struct net_device *dev,
3496                                              netdev_features_t features)
3497 {
3498         return vlan_features_check(skb, features);
3499 }
3500
3501 static netdev_features_t gso_features_check(const struct sk_buff *skb,
3502                                             struct net_device *dev,
3503                                             netdev_features_t features)
3504 {
3505         u16 gso_segs = skb_shinfo(skb)->gso_segs;
3506
3507         if (gso_segs > READ_ONCE(dev->gso_max_segs))
3508                 return features & ~NETIF_F_GSO_MASK;
3509
3510         if (!skb_shinfo(skb)->gso_type) {
3511                 skb_warn_bad_offload(skb);
3512                 return features & ~NETIF_F_GSO_MASK;
3513         }
3514
3515         /* Support for GSO partial features requires software
3516          * intervention before we can actually process the packets
3517          * so we need to strip support for any partial features now
3518          * and we can pull them back in after we have partially
3519          * segmented the frame.
3520          */
3521         if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3522                 features &= ~dev->gso_partial_features;
3523
3524         /* Make sure to clear the IPv4 ID mangling feature if the
3525          * IPv4 header has the potential to be fragmented.
3526          */
3527         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3528                 struct iphdr *iph = skb->encapsulation ?
3529                                     inner_ip_hdr(skb) : ip_hdr(skb);
3530
3531                 if (!(iph->frag_off & htons(IP_DF)))
3532                         features &= ~NETIF_F_TSO_MANGLEID;
3533         }
3534
3535         return features;
3536 }
3537
3538 netdev_features_t netif_skb_features(struct sk_buff *skb)
3539 {
3540         struct net_device *dev = skb->dev;
3541         netdev_features_t features = dev->features;
3542
3543         if (skb_is_gso(skb))
3544                 features = gso_features_check(skb, dev, features);
3545
3546         /* If encapsulation offload request, verify we are testing
3547          * hardware encapsulation features instead of standard
3548          * features for the netdev
3549          */
3550         if (skb->encapsulation)
3551                 features &= dev->hw_enc_features;
3552
3553         if (skb_vlan_tagged(skb))
3554                 features = netdev_intersect_features(features,
3555                                                      dev->vlan_features |
3556                                                      NETIF_F_HW_VLAN_CTAG_TX |
3557                                                      NETIF_F_HW_VLAN_STAG_TX);
3558
3559         if (dev->netdev_ops->ndo_features_check)
3560                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3561                                                                 features);
3562         else
3563                 features &= dflt_features_check(skb, dev, features);
3564
3565         return harmonize_features(skb, features);
3566 }
3567 EXPORT_SYMBOL(netif_skb_features);
3568
3569 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3570                     struct netdev_queue *txq, bool more)
3571 {
3572         unsigned int len;
3573         int rc;
3574
3575         if (dev_nit_active(dev))
3576                 dev_queue_xmit_nit(skb, dev);
3577
3578         len = skb->len;
3579         trace_net_dev_start_xmit(skb, dev);
3580         rc = netdev_start_xmit(skb, dev, txq, more);
3581         trace_net_dev_xmit(skb, rc, dev, len);
3582
3583         return rc;
3584 }
3585
3586 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3587                                     struct netdev_queue *txq, int *ret)
3588 {
3589         struct sk_buff *skb = first;
3590         int rc = NETDEV_TX_OK;
3591
3592         while (skb) {
3593                 struct sk_buff *next = skb->next;
3594
3595                 skb_mark_not_on_list(skb);
3596                 rc = xmit_one(skb, dev, txq, next != NULL);
3597                 if (unlikely(!dev_xmit_complete(rc))) {
3598                         skb->next = next;
3599                         goto out;
3600                 }
3601
3602                 skb = next;
3603                 if (netif_tx_queue_stopped(txq) && skb) {
3604                         rc = NETDEV_TX_BUSY;
3605                         break;
3606                 }
3607         }
3608
3609 out:
3610         *ret = rc;
3611         return skb;
3612 }
3613
3614 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3615                                           netdev_features_t features)
3616 {
3617         if (skb_vlan_tag_present(skb) &&
3618             !vlan_hw_offload_capable(features, skb->vlan_proto))
3619                 skb = __vlan_hwaccel_push_inside(skb);
3620         return skb;
3621 }
3622
3623 int skb_csum_hwoffload_help(struct sk_buff *skb,
3624                             const netdev_features_t features)
3625 {
3626         if (unlikely(skb_csum_is_sctp(skb)))
3627                 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3628                         skb_crc32c_csum_help(skb);
3629
3630         if (features & NETIF_F_HW_CSUM)
3631                 return 0;
3632
3633         if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3634                 switch (skb->csum_offset) {
3635                 case offsetof(struct tcphdr, check):
3636                 case offsetof(struct udphdr, check):
3637                         return 0;
3638                 }
3639         }
3640
3641         return skb_checksum_help(skb);
3642 }
3643 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3644
3645 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3646 {
3647         netdev_features_t features;
3648
3649         features = netif_skb_features(skb);
3650         skb = validate_xmit_vlan(skb, features);
3651         if (unlikely(!skb))
3652                 goto out_null;
3653
3654         skb = sk_validate_xmit_skb(skb, dev);
3655         if (unlikely(!skb))
3656                 goto out_null;
3657
3658         if (netif_needs_gso(skb, features)) {
3659                 struct sk_buff *segs;
3660
3661                 segs = skb_gso_segment(skb, features);
3662                 if (IS_ERR(segs)) {
3663                         goto out_kfree_skb;
3664                 } else if (segs) {
3665                         consume_skb(skb);
3666                         skb = segs;
3667                 }
3668         } else {
3669                 if (skb_needs_linearize(skb, features) &&
3670                     __skb_linearize(skb))
3671                         goto out_kfree_skb;
3672
3673                 /* If packet is not checksummed and device does not
3674                  * support checksumming for this protocol, complete
3675                  * checksumming here.
3676                  */
3677                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3678                         if (skb->encapsulation)
3679                                 skb_set_inner_transport_header(skb,
3680                                                                skb_checksum_start_offset(skb));
3681                         else
3682                                 skb_set_transport_header(skb,
3683                                                          skb_checksum_start_offset(skb));
3684                         if (skb_csum_hwoffload_help(skb, features))
3685                                 goto out_kfree_skb;
3686                 }
3687         }
3688
3689         skb = validate_xmit_xfrm(skb, features, again);
3690
3691         return skb;
3692
3693 out_kfree_skb:
3694         kfree_skb(skb);
3695 out_null:
3696         dev_core_stats_tx_dropped_inc(dev);
3697         return NULL;
3698 }
3699
3700 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3701 {
3702         struct sk_buff *next, *head = NULL, *tail;
3703
3704         for (; skb != NULL; skb = next) {
3705                 next = skb->next;
3706                 skb_mark_not_on_list(skb);
3707
3708                 /* in case skb wont be segmented, point to itself */
3709                 skb->prev = skb;
3710
3711                 skb = validate_xmit_skb(skb, dev, again);
3712                 if (!skb)
3713                         continue;
3714
3715                 if (!head)
3716                         head = skb;
3717                 else
3718                         tail->next = skb;
3719                 /* If skb was segmented, skb->prev points to
3720                  * the last segment. If not, it still contains skb.
3721                  */
3722                 tail = skb->prev;
3723         }
3724         return head;
3725 }
3726 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3727
3728 static void qdisc_pkt_len_init(struct sk_buff *skb)
3729 {
3730         const struct skb_shared_info *shinfo = skb_shinfo(skb);
3731
3732         qdisc_skb_cb(skb)->pkt_len = skb->len;
3733
3734         /* To get more precise estimation of bytes sent on wire,
3735          * we add to pkt_len the headers size of all segments
3736          */
3737         if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3738                 u16 gso_segs = shinfo->gso_segs;
3739                 unsigned int hdr_len;
3740
3741                 /* mac layer + network layer */
3742                 hdr_len = skb_transport_offset(skb);
3743
3744                 /* + transport layer */
3745                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3746                         const struct tcphdr *th;
3747                         struct tcphdr _tcphdr;
3748
3749                         th = skb_header_pointer(skb, hdr_len,
3750                                                 sizeof(_tcphdr), &_tcphdr);
3751                         if (likely(th))
3752                                 hdr_len += __tcp_hdrlen(th);
3753                 } else {
3754                         struct udphdr _udphdr;
3755
3756                         if (skb_header_pointer(skb, hdr_len,
3757                                                sizeof(_udphdr), &_udphdr))
3758                                 hdr_len += sizeof(struct udphdr);
3759                 }
3760
3761                 if (shinfo->gso_type & SKB_GSO_DODGY)
3762                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3763                                                 shinfo->gso_size);
3764
3765                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3766         }
3767 }
3768
3769 static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3770                              struct sk_buff **to_free,
3771                              struct netdev_queue *txq)
3772 {
3773         int rc;
3774
3775         rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3776         if (rc == NET_XMIT_SUCCESS)
3777                 trace_qdisc_enqueue(q, txq, skb);
3778         return rc;
3779 }
3780
3781 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3782                                  struct net_device *dev,
3783                                  struct netdev_queue *txq)
3784 {
3785         spinlock_t *root_lock = qdisc_lock(q);
3786         struct sk_buff *to_free = NULL;
3787         bool contended;
3788         int rc;
3789
3790         qdisc_calculate_pkt_len(skb, q);
3791
3792         if (q->flags & TCQ_F_NOLOCK) {
3793                 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3794                     qdisc_run_begin(q)) {
3795                         /* Retest nolock_qdisc_is_empty() within the protection
3796                          * of q->seqlock to protect from racing with requeuing.
3797                          */
3798                         if (unlikely(!nolock_qdisc_is_empty(q))) {
3799                                 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3800                                 __qdisc_run(q);
3801                                 qdisc_run_end(q);
3802
3803                                 goto no_lock_out;
3804                         }
3805
3806                         qdisc_bstats_cpu_update(q, skb);
3807                         if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3808                             !nolock_qdisc_is_empty(q))
3809                                 __qdisc_run(q);
3810
3811                         qdisc_run_end(q);
3812                         return NET_XMIT_SUCCESS;
3813                 }
3814
3815                 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3816                 qdisc_run(q);
3817
3818 no_lock_out:
3819                 if (unlikely(to_free))
3820                         kfree_skb_list_reason(to_free,
3821                                               SKB_DROP_REASON_QDISC_DROP);
3822                 return rc;
3823         }
3824
3825         /*
3826          * Heuristic to force contended enqueues to serialize on a
3827          * separate lock before trying to get qdisc main lock.
3828          * This permits qdisc->running owner to get the lock more
3829          * often and dequeue packets faster.
3830          * On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
3831          * and then other tasks will only enqueue packets. The packets will be
3832          * sent after the qdisc owner is scheduled again. To prevent this
3833          * scenario the task always serialize on the lock.
3834          */
3835         contended = qdisc_is_running(q) || IS_ENABLED(CONFIG_PREEMPT_RT);
3836         if (unlikely(contended))
3837                 spin_lock(&q->busylock);
3838
3839         spin_lock(root_lock);
3840         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3841                 __qdisc_drop(skb, &to_free);
3842                 rc = NET_XMIT_DROP;
3843         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3844                    qdisc_run_begin(q)) {
3845                 /*
3846                  * This is a work-conserving queue; there are no old skbs
3847                  * waiting to be sent out; and the qdisc is not running -
3848                  * xmit the skb directly.
3849                  */
3850
3851                 qdisc_bstats_update(q, skb);
3852
3853                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3854                         if (unlikely(contended)) {
3855                                 spin_unlock(&q->busylock);
3856                                 contended = false;
3857                         }
3858                         __qdisc_run(q);
3859                 }
3860
3861                 qdisc_run_end(q);
3862                 rc = NET_XMIT_SUCCESS;
3863         } else {
3864                 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3865                 if (qdisc_run_begin(q)) {
3866                         if (unlikely(contended)) {
3867                                 spin_unlock(&q->busylock);
3868                                 contended = false;
3869                         }
3870                         __qdisc_run(q);
3871                         qdisc_run_end(q);
3872                 }
3873         }
3874         spin_unlock(root_lock);
3875         if (unlikely(to_free))
3876                 kfree_skb_list_reason(to_free, SKB_DROP_REASON_QDISC_DROP);
3877         if (unlikely(contended))
3878                 spin_unlock(&q->busylock);
3879         return rc;
3880 }
3881
3882 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3883 static void skb_update_prio(struct sk_buff *skb)
3884 {
3885         const struct netprio_map *map;
3886         const struct sock *sk;
3887         unsigned int prioidx;
3888
3889         if (skb->priority)
3890                 return;
3891         map = rcu_dereference_bh(skb->dev->priomap);
3892         if (!map)
3893                 return;
3894         sk = skb_to_full_sk(skb);
3895         if (!sk)
3896                 return;
3897
3898         prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3899
3900         if (prioidx < map->priomap_len)
3901                 skb->priority = map->priomap[prioidx];
3902 }
3903 #else
3904 #define skb_update_prio(skb)
3905 #endif
3906
3907 /**
3908  *      dev_loopback_xmit - loop back @skb
3909  *      @net: network namespace this loopback is happening in
3910  *      @sk:  sk needed to be a netfilter okfn
3911  *      @skb: buffer to transmit
3912  */
3913 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3914 {
3915         skb_reset_mac_header(skb);
3916         __skb_pull(skb, skb_network_offset(skb));
3917         skb->pkt_type = PACKET_LOOPBACK;
3918         if (skb->ip_summed == CHECKSUM_NONE)
3919                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3920         DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
3921         skb_dst_force(skb);
3922         netif_rx(skb);
3923         return 0;
3924 }
3925 EXPORT_SYMBOL(dev_loopback_xmit);
3926
3927 #ifdef CONFIG_NET_EGRESS
3928 static struct sk_buff *
3929 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3930 {
3931 #ifdef CONFIG_NET_CLS_ACT
3932         struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3933         struct tcf_result cl_res;
3934
3935         if (!miniq)
3936                 return skb;
3937
3938         /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3939         tc_skb_cb(skb)->mru = 0;
3940         tc_skb_cb(skb)->post_ct = false;
3941         mini_qdisc_bstats_cpu_update(miniq, skb);
3942
3943         switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
3944         case TC_ACT_OK:
3945         case TC_ACT_RECLASSIFY:
3946                 skb->tc_index = TC_H_MIN(cl_res.classid);
3947                 break;
3948         case TC_ACT_SHOT:
3949                 mini_qdisc_qstats_cpu_drop(miniq);
3950                 *ret = NET_XMIT_DROP;
3951                 kfree_skb_reason(skb, SKB_DROP_REASON_TC_EGRESS);
3952                 return NULL;
3953         case TC_ACT_STOLEN:
3954         case TC_ACT_QUEUED:
3955         case TC_ACT_TRAP:
3956                 *ret = NET_XMIT_SUCCESS;
3957                 consume_skb(skb);
3958                 return NULL;
3959         case TC_ACT_REDIRECT:
3960                 /* No need to push/pop skb's mac_header here on egress! */
3961                 skb_do_redirect(skb);
3962                 *ret = NET_XMIT_SUCCESS;
3963                 return NULL;
3964         default:
3965                 break;
3966         }
3967 #endif /* CONFIG_NET_CLS_ACT */
3968
3969         return skb;
3970 }
3971
3972 static struct netdev_queue *
3973 netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
3974 {
3975         int qm = skb_get_queue_mapping(skb);
3976
3977         return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
3978 }
3979
3980 static bool netdev_xmit_txqueue_skipped(void)
3981 {
3982         return __this_cpu_read(softnet_data.xmit.skip_txqueue);
3983 }
3984
3985 void netdev_xmit_skip_txqueue(bool skip)
3986 {
3987         __this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
3988 }
3989 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
3990 #endif /* CONFIG_NET_EGRESS */
3991
3992 #ifdef CONFIG_XPS
3993 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
3994                                struct xps_dev_maps *dev_maps, unsigned int tci)
3995 {
3996         int tc = netdev_get_prio_tc_map(dev, skb->priority);
3997         struct xps_map *map;
3998         int queue_index = -1;
3999
4000         if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
4001                 return queue_index;
4002
4003         tci *= dev_maps->num_tc;
4004         tci += tc;
4005
4006         map = rcu_dereference(dev_maps->attr_map[tci]);
4007         if (map) {
4008                 if (map->len == 1)
4009                         queue_index = map->queues[0];
4010                 else
4011                         queue_index = map->queues[reciprocal_scale(
4012                                                 skb_get_hash(skb), map->len)];
4013                 if (unlikely(queue_index >= dev->real_num_tx_queues))
4014                         queue_index = -1;
4015         }
4016         return queue_index;
4017 }
4018 #endif
4019
4020 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4021                          struct sk_buff *skb)
4022 {
4023 #ifdef CONFIG_XPS
4024         struct xps_dev_maps *dev_maps;
4025         struct sock *sk = skb->sk;
4026         int queue_index = -1;
4027
4028         if (!static_key_false(&xps_needed))
4029                 return -1;
4030
4031         rcu_read_lock();
4032         if (!static_key_false(&xps_rxqs_needed))
4033                 goto get_cpus_map;
4034
4035         dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4036         if (dev_maps) {
4037                 int tci = sk_rx_queue_get(sk);
4038
4039                 if (tci >= 0)
4040                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4041                                                           tci);
4042         }
4043
4044 get_cpus_map:
4045         if (queue_index < 0) {
4046                 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4047                 if (dev_maps) {
4048                         unsigned int tci = skb->sender_cpu - 1;
4049
4050                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4051                                                           tci);
4052                 }
4053         }
4054         rcu_read_unlock();
4055
4056         return queue_index;
4057 #else
4058         return -1;
4059 #endif
4060 }
4061
4062 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4063                      struct net_device *sb_dev)
4064 {
4065         return 0;
4066 }
4067 EXPORT_SYMBOL(dev_pick_tx_zero);
4068
4069 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
4070                        struct net_device *sb_dev)
4071 {
4072         return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4073 }
4074 EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4075
4076 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4077                      struct net_device *sb_dev)
4078 {
4079         struct sock *sk = skb->sk;
4080         int queue_index = sk_tx_queue_get(sk);
4081
4082         sb_dev = sb_dev ? : dev;
4083
4084         if (queue_index < 0 || skb->ooo_okay ||
4085             queue_index >= dev->real_num_tx_queues) {
4086                 int new_index = get_xps_queue(dev, sb_dev, skb);
4087
4088                 if (new_index < 0)
4089                         new_index = skb_tx_hash(dev, sb_dev, skb);
4090
4091                 if (queue_index != new_index && sk &&
4092                     sk_fullsock(sk) &&
4093                     rcu_access_pointer(sk->sk_dst_cache))
4094                         sk_tx_queue_set(sk, new_index);
4095
4096                 queue_index = new_index;
4097         }
4098
4099         return queue_index;
4100 }
4101 EXPORT_SYMBOL(netdev_pick_tx);
4102
4103 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4104                                          struct sk_buff *skb,
4105                                          struct net_device *sb_dev)
4106 {
4107         int queue_index = 0;
4108
4109 #ifdef CONFIG_XPS
4110         u32 sender_cpu = skb->sender_cpu - 1;
4111
4112         if (sender_cpu >= (u32)NR_CPUS)
4113                 skb->sender_cpu = raw_smp_processor_id() + 1;
4114 #endif
4115
4116         if (dev->real_num_tx_queues != 1) {
4117                 const struct net_device_ops *ops = dev->netdev_ops;
4118
4119                 if (ops->ndo_select_queue)
4120                         queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4121                 else
4122                         queue_index = netdev_pick_tx(dev, skb, sb_dev);
4123
4124                 queue_index = netdev_cap_txqueue(dev, queue_index);
4125         }
4126
4127         skb_set_queue_mapping(skb, queue_index);
4128         return netdev_get_tx_queue(dev, queue_index);
4129 }
4130
4131 /**
4132  * __dev_queue_xmit() - transmit a buffer
4133  * @skb:        buffer to transmit
4134  * @sb_dev:     suboordinate device used for L2 forwarding offload
4135  *
4136  * Queue a buffer for transmission to a network device. The caller must
4137  * have set the device and priority and built the buffer before calling
4138  * this function. The function can be called from an interrupt.
4139  *
4140  * When calling this method, interrupts MUST be enabled. This is because
4141  * the BH enable code must have IRQs enabled so that it will not deadlock.
4142  *
4143  * Regardless of the return value, the skb is consumed, so it is currently
4144  * difficult to retry a send to this method. (You can bump the ref count
4145  * before sending to hold a reference for retry if you are careful.)
4146  *
4147  * Return:
4148  * * 0                          - buffer successfully transmitted
4149  * * positive qdisc return code - NET_XMIT_DROP etc.
4150  * * negative errno             - other errors
4151  */
4152 int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4153 {
4154         struct net_device *dev = skb->dev;
4155         struct netdev_queue *txq = NULL;
4156         struct Qdisc *q;
4157         int rc = -ENOMEM;
4158         bool again = false;
4159
4160         skb_reset_mac_header(skb);
4161         skb_assert_len(skb);
4162
4163         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4164                 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4165
4166         /* Disable soft irqs for various locks below. Also
4167          * stops preemption for RCU.
4168          */
4169         rcu_read_lock_bh();
4170
4171         skb_update_prio(skb);
4172
4173         qdisc_pkt_len_init(skb);
4174 #ifdef CONFIG_NET_CLS_ACT
4175         skb->tc_at_ingress = 0;
4176 #endif
4177 #ifdef CONFIG_NET_EGRESS
4178         if (static_branch_unlikely(&egress_needed_key)) {
4179                 if (nf_hook_egress_active()) {
4180                         skb = nf_hook_egress(skb, &rc, dev);
4181                         if (!skb)
4182                                 goto out;
4183                 }
4184
4185                 netdev_xmit_skip_txqueue(false);
4186
4187                 nf_skip_egress(skb, true);
4188                 skb = sch_handle_egress(skb, &rc, dev);
4189                 if (!skb)
4190                         goto out;
4191                 nf_skip_egress(skb, false);
4192
4193                 if (netdev_xmit_txqueue_skipped())
4194                         txq = netdev_tx_queue_mapping(dev, skb);
4195         }
4196 #endif
4197         /* If device/qdisc don't need skb->dst, release it right now while
4198          * its hot in this cpu cache.
4199          */
4200         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4201                 skb_dst_drop(skb);
4202         else
4203                 skb_dst_force(skb);
4204
4205         if (!txq)
4206                 txq = netdev_core_pick_tx(dev, skb, sb_dev);
4207
4208         q = rcu_dereference_bh(txq->qdisc);
4209
4210         trace_net_dev_queue(skb);
4211         if (q->enqueue) {
4212                 rc = __dev_xmit_skb(skb, q, dev, txq);
4213                 goto out;
4214         }
4215
4216         /* The device has no queue. Common case for software devices:
4217          * loopback, all the sorts of tunnels...
4218
4219          * Really, it is unlikely that netif_tx_lock protection is necessary
4220          * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
4221          * counters.)
4222          * However, it is possible, that they rely on protection
4223          * made by us here.
4224
4225          * Check this and shot the lock. It is not prone from deadlocks.
4226          *Either shot noqueue qdisc, it is even simpler 8)
4227          */
4228         if (dev->flags & IFF_UP) {
4229                 int cpu = smp_processor_id(); /* ok because BHs are off */
4230
4231                 /* Other cpus might concurrently change txq->xmit_lock_owner
4232                  * to -1 or to their cpu id, but not to our id.
4233                  */
4234                 if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
4235                         if (dev_xmit_recursion())
4236                                 goto recursion_alert;
4237
4238                         skb = validate_xmit_skb(skb, dev, &again);
4239                         if (!skb)
4240                                 goto out;
4241
4242                         HARD_TX_LOCK(dev, txq, cpu);
4243
4244                         if (!netif_xmit_stopped(txq)) {
4245                                 dev_xmit_recursion_inc();
4246                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4247                                 dev_xmit_recursion_dec();
4248                                 if (dev_xmit_complete(rc)) {
4249                                         HARD_TX_UNLOCK(dev, txq);
4250                                         goto out;
4251                                 }
4252                         }
4253                         HARD_TX_UNLOCK(dev, txq);
4254                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4255                                              dev->name);
4256                 } else {
4257                         /* Recursion is detected! It is possible,
4258                          * unfortunately
4259                          */
4260 recursion_alert:
4261                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4262                                              dev->name);
4263                 }
4264         }
4265
4266         rc = -ENETDOWN;
4267         rcu_read_unlock_bh();
4268
4269         dev_core_stats_tx_dropped_inc(dev);
4270         kfree_skb_list(skb);
4271         return rc;
4272 out:
4273         rcu_read_unlock_bh();
4274         return rc;
4275 }
4276 EXPORT_SYMBOL(__dev_queue_xmit);
4277
4278 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4279 {
4280         struct net_device *dev = skb->dev;
4281         struct sk_buff *orig_skb = skb;
4282         struct netdev_queue *txq;
4283         int ret = NETDEV_TX_BUSY;
4284         bool again = false;
4285
4286         if (unlikely(!netif_running(dev) ||
4287                      !netif_carrier_ok(dev)))
4288                 goto drop;
4289
4290         skb = validate_xmit_skb_list(skb, dev, &again);
4291         if (skb != orig_skb)
4292                 goto drop;
4293
4294         skb_set_queue_mapping(skb, queue_id);
4295         txq = skb_get_tx_queue(dev, skb);
4296
4297         local_bh_disable();
4298
4299         dev_xmit_recursion_inc();
4300         HARD_TX_LOCK(dev, txq, smp_processor_id());
4301         if (!netif_xmit_frozen_or_drv_stopped(txq))
4302                 ret = netdev_start_xmit(skb, dev, txq, false);
4303         HARD_TX_UNLOCK(dev, txq);
4304         dev_xmit_recursion_dec();
4305
4306         local_bh_enable();
4307         return ret;
4308 drop:
4309         dev_core_stats_tx_dropped_inc(dev);
4310         kfree_skb_list(skb);
4311         return NET_XMIT_DROP;
4312 }
4313 EXPORT_SYMBOL(__dev_direct_xmit);
4314
4315 /*************************************************************************
4316  *                      Receiver routines
4317  *************************************************************************/
4318
4319 int netdev_max_backlog __read_mostly = 1000;
4320 EXPORT_SYMBOL(netdev_max_backlog);
4321
4322 int netdev_tstamp_prequeue __read_mostly = 1;
4323 unsigned int sysctl_skb_defer_max __read_mostly = 64;
4324 int netdev_budget __read_mostly = 300;
4325 /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4326 unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
4327 int weight_p __read_mostly = 64;           /* old backlog weight */
4328 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
4329 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
4330 int dev_rx_weight __read_mostly = 64;
4331 int dev_tx_weight __read_mostly = 64;
4332
4333 /* Called with irq disabled */
4334 static inline void ____napi_schedule(struct softnet_data *sd,
4335                                      struct napi_struct *napi)
4336 {
4337         struct task_struct *thread;
4338
4339         lockdep_assert_irqs_disabled();
4340
4341         if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4342                 /* Paired with smp_mb__before_atomic() in
4343                  * napi_enable()/dev_set_threaded().
4344                  * Use READ_ONCE() to guarantee a complete
4345                  * read on napi->thread. Only call
4346                  * wake_up_process() when it's not NULL.
4347                  */
4348                 thread = READ_ONCE(napi->thread);
4349                 if (thread) {
4350                         /* Avoid doing set_bit() if the thread is in
4351                          * INTERRUPTIBLE state, cause napi_thread_wait()
4352                          * makes sure to proceed with napi polling
4353                          * if the thread is explicitly woken from here.
4354                          */
4355                         if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4356                                 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4357                         wake_up_process(thread);
4358                         return;
4359                 }
4360         }
4361
4362         list_add_tail(&napi->poll_list, &sd->poll_list);
4363         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4364 }
4365
4366 #ifdef CONFIG_RPS
4367
4368 /* One global table that all flow-based protocols share. */
4369 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4370 EXPORT_SYMBOL(rps_sock_flow_table);
4371 u32 rps_cpu_mask __read_mostly;
4372 EXPORT_SYMBOL(rps_cpu_mask);
4373
4374 struct static_key_false rps_needed __read_mostly;
4375 EXPORT_SYMBOL(rps_needed);
4376 struct static_key_false rfs_needed __read_mostly;
4377 EXPORT_SYMBOL(rfs_needed);
4378
4379 static struct rps_dev_flow *
4380 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4381             struct rps_dev_flow *rflow, u16 next_cpu)
4382 {
4383         if (next_cpu < nr_cpu_ids) {
4384 #ifdef CONFIG_RFS_ACCEL
4385                 struct netdev_rx_queue *rxqueue;
4386                 struct rps_dev_flow_table *flow_table;
4387                 struct rps_dev_flow *old_rflow;
4388                 u32 flow_id;
4389                 u16 rxq_index;
4390                 int rc;
4391
4392                 /* Should we steer this flow to a different hardware queue? */
4393                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4394                     !(dev->features & NETIF_F_NTUPLE))
4395                         goto out;
4396                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4397                 if (rxq_index == skb_get_rx_queue(skb))
4398                         goto out;
4399
4400                 rxqueue = dev->_rx + rxq_index;
4401                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4402                 if (!flow_table)
4403                         goto out;
4404                 flow_id = skb_get_hash(skb) & flow_table->mask;
4405                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4406                                                         rxq_index, flow_id);
4407                 if (rc < 0)
4408                         goto out;
4409                 old_rflow = rflow;
4410                 rflow = &flow_table->flows[flow_id];
4411                 rflow->filter = rc;
4412                 if (old_rflow->filter == rflow->filter)
4413                         old_rflow->filter = RPS_NO_FILTER;
4414         out:
4415 #endif
4416                 rflow->last_qtail =
4417                         per_cpu(softnet_data, next_cpu).input_queue_head;
4418         }
4419
4420         rflow->cpu = next_cpu;
4421         return rflow;
4422 }
4423
4424 /*
4425  * get_rps_cpu is called from netif_receive_skb and returns the target
4426  * CPU from the RPS map of the receiving queue for a given skb.
4427  * rcu_read_lock must be held on entry.
4428  */
4429 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4430                        struct rps_dev_flow **rflowp)
4431 {
4432         const struct rps_sock_flow_table *sock_flow_table;
4433         struct netdev_rx_queue *rxqueue = dev->_rx;
4434         struct rps_dev_flow_table *flow_table;
4435         struct rps_map *map;
4436         int cpu = -1;
4437         u32 tcpu;
4438         u32 hash;
4439
4440         if (skb_rx_queue_recorded(skb)) {
4441                 u16 index = skb_get_rx_queue(skb);
4442
4443                 if (unlikely(index >= dev->real_num_rx_queues)) {
4444                         WARN_ONCE(dev->real_num_rx_queues > 1,
4445                                   "%s received packet on queue %u, but number "
4446                                   "of RX queues is %u\n",
4447                                   dev->name, index, dev->real_num_rx_queues);
4448                         goto done;
4449                 }
4450                 rxqueue += index;
4451         }
4452
4453         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4454
4455         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4456         map = rcu_dereference(rxqueue->rps_map);
4457         if (!flow_table && !map)
4458                 goto done;
4459
4460         skb_reset_network_header(skb);
4461         hash = skb_get_hash(skb);
4462         if (!hash)
4463                 goto done;
4464
4465         sock_flow_table = rcu_dereference(rps_sock_flow_table);
4466         if (flow_table && sock_flow_table) {
4467                 struct rps_dev_flow *rflow;
4468                 u32 next_cpu;
4469                 u32 ident;
4470
4471                 /* First check into global flow table if there is a match */
4472                 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4473                 if ((ident ^ hash) & ~rps_cpu_mask)
4474                         goto try_rps;
4475
4476                 next_cpu = ident & rps_cpu_mask;
4477
4478                 /* OK, now we know there is a match,
4479                  * we can look at the local (per receive queue) flow table
4480                  */
4481                 rflow = &flow_table->flows[hash & flow_table->mask];
4482                 tcpu = rflow->cpu;
4483
4484                 /*
4485                  * If the desired CPU (where last recvmsg was done) is
4486                  * different from current CPU (one in the rx-queue flow
4487                  * table entry), switch if one of the following holds:
4488                  *   - Current CPU is unset (>= nr_cpu_ids).
4489                  *   - Current CPU is offline.
4490                  *   - The current CPU's queue tail has advanced beyond the
4491                  *     last packet that was enqueued using this table entry.
4492                  *     This guarantees that all previous packets for the flow
4493                  *     have been dequeued, thus preserving in order delivery.
4494                  */
4495                 if (unlikely(tcpu != next_cpu) &&
4496                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4497                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4498                       rflow->last_qtail)) >= 0)) {
4499                         tcpu = next_cpu;
4500                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4501                 }
4502
4503                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4504                         *rflowp = rflow;
4505                         cpu = tcpu;
4506                         goto done;
4507                 }
4508         }
4509
4510 try_rps:
4511
4512         if (map) {
4513                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4514                 if (cpu_online(tcpu)) {
4515                         cpu = tcpu;
4516                         goto done;
4517                 }
4518         }
4519
4520 done:
4521         return cpu;
4522 }
4523
4524 #ifdef CONFIG_RFS_ACCEL
4525
4526 /**
4527  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4528  * @dev: Device on which the filter was set
4529  * @rxq_index: RX queue index
4530  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4531  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4532  *
4533  * Drivers that implement ndo_rx_flow_steer() should periodically call
4534  * this function for each installed filter and remove the filters for
4535  * which it returns %true.
4536  */
4537 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4538                          u32 flow_id, u16 filter_id)
4539 {
4540         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4541         struct rps_dev_flow_table *flow_table;
4542         struct rps_dev_flow *rflow;
4543         bool expire = true;
4544         unsigned int cpu;
4545
4546         rcu_read_lock();
4547         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4548         if (flow_table && flow_id <= flow_table->mask) {
4549                 rflow = &flow_table->flows[flow_id];
4550                 cpu = READ_ONCE(rflow->cpu);
4551                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4552                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4553                            rflow->last_qtail) <
4554                      (int)(10 * flow_table->mask)))
4555                         expire = false;
4556         }
4557         rcu_read_unlock();
4558         return expire;
4559 }
4560 EXPORT_SYMBOL(rps_may_expire_flow);
4561
4562 #endif /* CONFIG_RFS_ACCEL */
4563
4564 /* Called from hardirq (IPI) context */
4565 static void rps_trigger_softirq(void *data)
4566 {
4567         struct softnet_data *sd = data;
4568
4569         ____napi_schedule(sd, &sd->backlog);
4570         sd->received_rps++;
4571 }
4572
4573 #endif /* CONFIG_RPS */
4574
4575 /* Called from hardirq (IPI) context */
4576 static void trigger_rx_softirq(void *data)
4577 {
4578         struct softnet_data *sd = data;
4579
4580         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4581         smp_store_release(&sd->defer_ipi_scheduled, 0);
4582 }
4583
4584 /*
4585  * After we queued a packet into sd->input_pkt_queue,
4586  * we need to make sure this queue is serviced soon.
4587  *
4588  * - If this is another cpu queue, link it to our rps_ipi_list,
4589  *   and make sure we will process rps_ipi_list from net_rx_action().
4590  *   As we do not know yet if we are called from net_rx_action(),
4591  *   we have to raise NET_RX_SOFTIRQ. This might change in the future.
4592  *
4593  * - If this is our own queue, NAPI schedule our backlog.
4594  *   Note that this also raises NET_RX_SOFTIRQ.
4595  */
4596 static void napi_schedule_rps(struct softnet_data *sd)
4597 {
4598         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4599
4600 #ifdef CONFIG_RPS
4601         if (sd != mysd) {
4602                 sd->rps_ipi_next = mysd->rps_ipi_list;
4603                 mysd->rps_ipi_list = sd;
4604
4605                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4606                 return;
4607         }
4608 #endif /* CONFIG_RPS */
4609         __napi_schedule_irqoff(&mysd->backlog);
4610 }
4611
4612 #ifdef CONFIG_NET_FLOW_LIMIT
4613 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4614 #endif
4615
4616 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4617 {
4618 #ifdef CONFIG_NET_FLOW_LIMIT
4619         struct sd_flow_limit *fl;
4620         struct softnet_data *sd;
4621         unsigned int old_flow, new_flow;
4622
4623         if (qlen < (READ_ONCE(netdev_max_backlog) >> 1))
4624                 return false;
4625
4626         sd = this_cpu_ptr(&softnet_data);
4627
4628         rcu_read_lock();
4629         fl = rcu_dereference(sd->flow_limit);
4630         if (fl) {
4631                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4632                 old_flow = fl->history[fl->history_head];
4633                 fl->history[fl->history_head] = new_flow;
4634
4635                 fl->history_head++;
4636                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4637
4638                 if (likely(fl->buckets[old_flow]))
4639                         fl->buckets[old_flow]--;
4640
4641                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4642                         fl->count++;
4643                         rcu_read_unlock();
4644                         return true;
4645                 }
4646         }
4647         rcu_read_unlock();
4648 #endif
4649         return false;
4650 }
4651
4652 /*
4653  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4654  * queue (may be a remote CPU queue).
4655  */
4656 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4657                               unsigned int *qtail)
4658 {
4659         enum skb_drop_reason reason;
4660         struct softnet_data *sd;
4661         unsigned long flags;
4662         unsigned int qlen;
4663
4664         reason = SKB_DROP_REASON_NOT_SPECIFIED;
4665         sd = &per_cpu(softnet_data, cpu);
4666
4667         rps_lock_irqsave(sd, &flags);
4668         if (!netif_running(skb->dev))
4669                 goto drop;
4670         qlen = skb_queue_len(&sd->input_pkt_queue);
4671         if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) {
4672                 if (qlen) {
4673 enqueue:
4674                         __skb_queue_tail(&sd->input_pkt_queue, skb);
4675                         input_queue_tail_incr_save(sd, qtail);
4676                         rps_unlock_irq_restore(sd, &flags);
4677                         return NET_RX_SUCCESS;
4678                 }
4679
4680                 /* Schedule NAPI for backlog device
4681                  * We can use non atomic operation since we own the queue lock
4682                  */
4683                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
4684                         napi_schedule_rps(sd);
4685                 goto enqueue;
4686         }
4687         reason = SKB_DROP_REASON_CPU_BACKLOG;
4688
4689 drop:
4690         sd->dropped++;
4691         rps_unlock_irq_restore(sd, &flags);
4692
4693         dev_core_stats_rx_dropped_inc(skb->dev);
4694         kfree_skb_reason(skb, reason);
4695         return NET_RX_DROP;
4696 }
4697
4698 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4699 {
4700         struct net_device *dev = skb->dev;
4701         struct netdev_rx_queue *rxqueue;
4702
4703         rxqueue = dev->_rx;
4704
4705         if (skb_rx_queue_recorded(skb)) {
4706                 u16 index = skb_get_rx_queue(skb);
4707
4708                 if (unlikely(index >= dev->real_num_rx_queues)) {
4709                         WARN_ONCE(dev->real_num_rx_queues > 1,
4710                                   "%s received packet on queue %u, but number "
4711                                   "of RX queues is %u\n",
4712                                   dev->name, index, dev->real_num_rx_queues);
4713
4714                         return rxqueue; /* Return first rxqueue */
4715                 }
4716                 rxqueue += index;
4717         }
4718         return rxqueue;
4719 }
4720
4721 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4722                              struct bpf_prog *xdp_prog)
4723 {
4724         void *orig_data, *orig_data_end, *hard_start;
4725         struct netdev_rx_queue *rxqueue;
4726         bool orig_bcast, orig_host;
4727         u32 mac_len, frame_sz;
4728         __be16 orig_eth_type;
4729         struct ethhdr *eth;
4730         u32 metalen, act;
4731         int off;
4732
4733         /* The XDP program wants to see the packet starting at the MAC
4734          * header.
4735          */
4736         mac_len = skb->data - skb_mac_header(skb);
4737         hard_start = skb->data - skb_headroom(skb);
4738
4739         /* SKB "head" area always have tailroom for skb_shared_info */
4740         frame_sz = (void *)skb_end_pointer(skb) - hard_start;
4741         frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4742
4743         rxqueue = netif_get_rxqueue(skb);
4744         xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4745         xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4746                          skb_headlen(skb) + mac_len, true);
4747
4748         orig_data_end = xdp->data_end;
4749         orig_data = xdp->data;
4750         eth = (struct ethhdr *)xdp->data;
4751         orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
4752         orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4753         orig_eth_type = eth->h_proto;
4754
4755         act = bpf_prog_run_xdp(xdp_prog, xdp);
4756
4757         /* check if bpf_xdp_adjust_head was used */
4758         off = xdp->data - orig_data;
4759         if (off) {
4760                 if (off > 0)
4761                         __skb_pull(skb, off);
4762                 else if (off < 0)
4763                         __skb_push(skb, -off);
4764
4765                 skb->mac_header += off;
4766                 skb_reset_network_header(skb);
4767         }
4768
4769         /* check if bpf_xdp_adjust_tail was used */
4770         off = xdp->data_end - orig_data_end;
4771         if (off != 0) {
4772                 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4773                 skb->len += off; /* positive on grow, negative on shrink */
4774         }
4775
4776         /* check if XDP changed eth hdr such SKB needs update */
4777         eth = (struct ethhdr *)xdp->data;
4778         if ((orig_eth_type != eth->h_proto) ||
4779             (orig_host != ether_addr_equal_64bits(eth->h_dest,
4780                                                   skb->dev->dev_addr)) ||
4781             (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4782                 __skb_push(skb, ETH_HLEN);
4783                 skb->pkt_type = PACKET_HOST;
4784                 skb->protocol = eth_type_trans(skb, skb->dev);
4785         }
4786
4787         /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
4788          * before calling us again on redirect path. We do not call do_redirect
4789          * as we leave that up to the caller.
4790          *
4791          * Caller is responsible for managing lifetime of skb (i.e. calling
4792          * kfree_skb in response to actions it cannot handle/XDP_DROP).
4793          */
4794         switch (act) {
4795         case XDP_REDIRECT:
4796         case XDP_TX:
4797                 __skb_push(skb, mac_len);
4798                 break;
4799         case XDP_PASS:
4800                 metalen = xdp->data - xdp->data_meta;
4801                 if (metalen)
4802                         skb_metadata_set(skb, metalen);
4803                 break;
4804         }
4805
4806         return act;
4807 }
4808
4809 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4810                                      struct xdp_buff *xdp,
4811                                      struct bpf_prog *xdp_prog)
4812 {
4813         u32 act = XDP_DROP;
4814
4815         /* Reinjected packets coming from act_mirred or similar should
4816          * not get XDP generic processing.
4817          */
4818         if (skb_is_redirected(skb))
4819                 return XDP_PASS;
4820
4821         /* XDP packets must be linear and must have sufficient headroom
4822          * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4823          * native XDP provides, thus we need to do it here as well.
4824          */
4825         if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4826             skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4827                 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4828                 int troom = skb->tail + skb->data_len - skb->end;
4829
4830                 /* In case we have to go down the path and also linearize,
4831                  * then lets do the pskb_expand_head() work just once here.
4832                  */
4833                 if (pskb_expand_head(skb,
4834                                      hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4835                                      troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4836                         goto do_drop;
4837                 if (skb_linearize(skb))
4838                         goto do_drop;
4839         }
4840
4841         act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4842         switch (act) {
4843         case XDP_REDIRECT:
4844         case XDP_TX:
4845         case XDP_PASS:
4846                 break;
4847         default:
4848                 bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act);
4849                 fallthrough;
4850         case XDP_ABORTED:
4851                 trace_xdp_exception(skb->dev, xdp_prog, act);
4852                 fallthrough;
4853         case XDP_DROP:
4854         do_drop:
4855                 kfree_skb(skb);
4856                 break;
4857         }
4858
4859         return act;
4860 }
4861
4862 /* When doing generic XDP we have to bypass the qdisc layer and the
4863  * network taps in order to match in-driver-XDP behavior. This also means
4864  * that XDP packets are able to starve other packets going through a qdisc,
4865  * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
4866  * queues, so they do not have this starvation issue.
4867  */
4868 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4869 {
4870         struct net_device *dev = skb->dev;
4871         struct netdev_queue *txq;
4872         bool free_skb = true;
4873         int cpu, rc;
4874
4875         txq = netdev_core_pick_tx(dev, skb, NULL);
4876         cpu = smp_processor_id();
4877         HARD_TX_LOCK(dev, txq, cpu);
4878         if (!netif_xmit_frozen_or_drv_stopped(txq)) {
4879                 rc = netdev_start_xmit(skb, dev, txq, 0);
4880                 if (dev_xmit_complete(rc))
4881                         free_skb = false;
4882         }
4883         HARD_TX_UNLOCK(dev, txq);
4884         if (free_skb) {
4885                 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4886                 dev_core_stats_tx_dropped_inc(dev);
4887                 kfree_skb(skb);
4888         }
4889 }
4890
4891 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4892
4893 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4894 {
4895         if (xdp_prog) {
4896                 struct xdp_buff xdp;
4897                 u32 act;
4898                 int err;
4899
4900                 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4901                 if (act != XDP_PASS) {
4902                         switch (act) {
4903                         case XDP_REDIRECT:
4904                                 err = xdp_do_generic_redirect(skb->dev, skb,
4905                                                               &xdp, xdp_prog);
4906                                 if (err)
4907                                         goto out_redir;
4908                                 break;
4909                         case XDP_TX:
4910                                 generic_xdp_tx(skb, xdp_prog);
4911                                 break;
4912                         }
4913                         return XDP_DROP;
4914                 }
4915         }
4916         return XDP_PASS;
4917 out_redir:
4918         kfree_skb_reason(skb, SKB_DROP_REASON_XDP);
4919         return XDP_DROP;
4920 }
4921 EXPORT_SYMBOL_GPL(do_xdp_generic);
4922
4923 static int netif_rx_internal(struct sk_buff *skb)
4924 {
4925         int ret;
4926
4927         net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
4928
4929         trace_netif_rx(skb);
4930
4931 #ifdef CONFIG_RPS
4932         if (static_branch_unlikely(&rps_needed)) {
4933                 struct rps_dev_flow voidflow, *rflow = &voidflow;
4934                 int cpu;
4935
4936                 rcu_read_lock();
4937
4938                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
4939                 if (cpu < 0)
4940                         cpu = smp_processor_id();
4941
4942                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4943
4944                 rcu_read_unlock();
4945         } else
4946 #endif
4947         {
4948                 unsigned int qtail;
4949
4950                 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
4951         }
4952         return ret;
4953 }
4954
4955 /**
4956  *      __netif_rx      -       Slightly optimized version of netif_rx
4957  *      @skb: buffer to post
4958  *
4959  *      This behaves as netif_rx except that it does not disable bottom halves.
4960  *      As a result this function may only be invoked from the interrupt context
4961  *      (either hard or soft interrupt).
4962  */
4963 int __netif_rx(struct sk_buff *skb)
4964 {
4965         int ret;
4966
4967         lockdep_assert_once(hardirq_count() | softirq_count());
4968
4969         trace_netif_rx_entry(skb);
4970         ret = netif_rx_internal(skb);
4971         trace_netif_rx_exit(ret);
4972         return ret;
4973 }
4974 EXPORT_SYMBOL(__netif_rx);
4975
4976 /**
4977  *      netif_rx        -       post buffer to the network code
4978  *      @skb: buffer to post
4979  *
4980  *      This function receives a packet from a device driver and queues it for
4981  *      the upper (protocol) levels to process via the backlog NAPI device. It
4982  *      always succeeds. The buffer may be dropped during processing for
4983  *      congestion control or by the protocol layers.
4984  *      The network buffer is passed via the backlog NAPI device. Modern NIC
4985  *      driver should use NAPI and GRO.
4986  *      This function can used from interrupt and from process context. The
4987  *      caller from process context must not disable interrupts before invoking
4988  *      this function.
4989  *
4990  *      return values:
4991  *      NET_RX_SUCCESS  (no congestion)
4992  *      NET_RX_DROP     (packet was dropped)
4993  *
4994  */
4995 int netif_rx(struct sk_buff *skb)
4996 {
4997         bool need_bh_off = !(hardirq_count() | softirq_count());
4998         int ret;
4999
5000         if (need_bh_off)
5001                 local_bh_disable();
5002         trace_netif_rx_entry(skb);
5003         ret = netif_rx_internal(skb);
5004         trace_netif_rx_exit(ret);
5005         if (need_bh_off)
5006                 local_bh_enable();
5007         return ret;
5008 }
5009 EXPORT_SYMBOL(netif_rx);
5010
5011 static __latent_entropy void net_tx_action(struct softirq_action *h)
5012 {
5013         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5014
5015         if (sd->completion_queue) {
5016                 struct sk_buff *clist;
5017
5018                 local_irq_disable();
5019                 clist = sd->completion_queue;
5020                 sd->completion_queue = NULL;
5021                 local_irq_enable();
5022
5023                 while (clist) {
5024                         struct sk_buff *skb = clist;
5025
5026                         clist = clist->next;
5027
5028                         WARN_ON(refcount_read(&skb->users));
5029                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED))
5030                                 trace_consume_skb(skb, net_tx_action);
5031                         else
5032                                 trace_kfree_skb(skb, net_tx_action,
5033                                                 get_kfree_skb_cb(skb)->reason);
5034
5035                         if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5036                                 __kfree_skb(skb);
5037                         else
5038                                 __kfree_skb_defer(skb);
5039                 }
5040         }
5041
5042         if (sd->output_queue) {
5043                 struct Qdisc *head;
5044
5045                 local_irq_disable();
5046                 head = sd->output_queue;
5047                 sd->output_queue = NULL;
5048                 sd->output_queue_tailp = &sd->output_queue;
5049                 local_irq_enable();
5050
5051                 rcu_read_lock();
5052
5053                 while (head) {
5054                         struct Qdisc *q = head;
5055                         spinlock_t *root_lock = NULL;
5056
5057                         head = head->next_sched;
5058
5059                         /* We need to make sure head->next_sched is read
5060                          * before clearing __QDISC_STATE_SCHED
5061                          */
5062                         smp_mb__before_atomic();
5063
5064                         if (!(q->flags & TCQ_F_NOLOCK)) {
5065                                 root_lock = qdisc_lock(q);
5066                                 spin_lock(root_lock);
5067                         } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5068                                                      &q->state))) {
5069                                 /* There is a synchronize_net() between
5070                                  * STATE_DEACTIVATED flag being set and
5071                                  * qdisc_reset()/some_qdisc_is_busy() in
5072                                  * dev_deactivate(), so we can safely bail out
5073                                  * early here to avoid data race between
5074                                  * qdisc_deactivate() and some_qdisc_is_busy()
5075                                  * for lockless qdisc.
5076                                  */
5077                                 clear_bit(__QDISC_STATE_SCHED, &q->state);
5078                                 continue;
5079                         }
5080
5081                         clear_bit(__QDISC_STATE_SCHED, &q->state);
5082                         qdisc_run(q);
5083                         if (root_lock)
5084                                 spin_unlock(root_lock);
5085                 }
5086
5087                 rcu_read_unlock();
5088         }
5089
5090         xfrm_dev_backlog(sd);
5091 }
5092
5093 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
5094 /* This hook is defined here for ATM LANE */
5095 int (*br_fdb_test_addr_hook)(struct net_device *dev,
5096                              unsigned char *addr) __read_mostly;
5097 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
5098 #endif
5099
5100 static inline struct sk_buff *
5101 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
5102                    struct net_device *orig_dev, bool *another)
5103 {
5104 #ifdef CONFIG_NET_CLS_ACT
5105         struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
5106         struct tcf_result cl_res;
5107
5108         /* If there's at least one ingress present somewhere (so
5109          * we get here via enabled static key), remaining devices
5110          * that are not configured with an ingress qdisc will bail
5111          * out here.
5112          */
5113         if (!miniq)
5114                 return skb;
5115
5116         if (*pt_prev) {
5117                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5118                 *pt_prev = NULL;
5119         }
5120
5121         qdisc_skb_cb(skb)->pkt_len = skb->len;
5122         tc_skb_cb(skb)->mru = 0;
5123         tc_skb_cb(skb)->post_ct = false;
5124         skb->tc_at_ingress = 1;
5125         mini_qdisc_bstats_cpu_update(miniq, skb);
5126
5127         switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
5128         case TC_ACT_OK:
5129         case TC_ACT_RECLASSIFY:
5130                 skb->tc_index = TC_H_MIN(cl_res.classid);
5131                 break;
5132         case TC_ACT_SHOT:
5133                 mini_qdisc_qstats_cpu_drop(miniq);
5134                 kfree_skb_reason(skb, SKB_DROP_REASON_TC_INGRESS);
5135                 *ret = NET_RX_DROP;
5136                 return NULL;
5137         case TC_ACT_STOLEN:
5138         case TC_ACT_QUEUED:
5139         case TC_ACT_TRAP:
5140                 consume_skb(skb);
5141                 *ret = NET_RX_SUCCESS;
5142                 return NULL;
5143         case TC_ACT_REDIRECT:
5144                 /* skb_mac_header check was done by cls/act_bpf, so
5145                  * we can safely push the L2 header back before
5146                  * redirecting to another netdev
5147                  */
5148                 __skb_push(skb, skb->mac_len);
5149                 if (skb_do_redirect(skb) == -EAGAIN) {
5150                         __skb_pull(skb, skb->mac_len);
5151                         *another = true;
5152                         break;
5153                 }
5154                 *ret = NET_RX_SUCCESS;
5155                 return NULL;
5156         case TC_ACT_CONSUMED:
5157                 *ret = NET_RX_SUCCESS;
5158                 return NULL;
5159         default:
5160                 break;
5161         }
5162 #endif /* CONFIG_NET_CLS_ACT */
5163         return skb;
5164 }
5165
5166 /**
5167  *      netdev_is_rx_handler_busy - check if receive handler is registered
5168  *      @dev: device to check
5169  *
5170  *      Check if a receive handler is already registered for a given device.
5171  *      Return true if there one.
5172  *
5173  *      The caller must hold the rtnl_mutex.
5174  */
5175 bool netdev_is_rx_handler_busy(struct net_device *dev)
5176 {
5177         ASSERT_RTNL();
5178         return dev && rtnl_dereference(dev->rx_handler);
5179 }
5180 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5181
5182 /**
5183  *      netdev_rx_handler_register - register receive handler
5184  *      @dev: device to register a handler for
5185  *      @rx_handler: receive handler to register
5186  *      @rx_handler_data: data pointer that is used by rx handler
5187  *
5188  *      Register a receive handler for a device. This handler will then be
5189  *      called from __netif_receive_skb. A negative errno code is returned
5190  *      on a failure.
5191  *
5192  *      The caller must hold the rtnl_mutex.
5193  *
5194  *      For a general description of rx_handler, see enum rx_handler_result.
5195  */
5196 int netdev_rx_handler_register(struct net_device *dev,
5197                                rx_handler_func_t *rx_handler,
5198                                void *rx_handler_data)
5199 {
5200         if (netdev_is_rx_handler_busy(dev))
5201                 return -EBUSY;
5202
5203         if (dev->priv_flags & IFF_NO_RX_HANDLER)
5204                 return -EINVAL;
5205
5206         /* Note: rx_handler_data must be set before rx_handler */
5207         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
5208         rcu_assign_pointer(dev->rx_handler, rx_handler);
5209
5210         return 0;
5211 }
5212 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5213
5214 /**
5215  *      netdev_rx_handler_unregister - unregister receive handler
5216  *      @dev: device to unregister a handler from
5217  *
5218  *      Unregister a receive handler from a device.
5219  *
5220  *      The caller must hold the rtnl_mutex.
5221  */
5222 void netdev_rx_handler_unregister(struct net_device *dev)
5223 {
5224
5225         ASSERT_RTNL();
5226         RCU_INIT_POINTER(dev->rx_handler, NULL);
5227         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5228          * section has a guarantee to see a non NULL rx_handler_data
5229          * as well.
5230          */
5231         synchronize_net();
5232         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
5233 }
5234 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5235
5236 /*
5237  * Limit the use of PFMEMALLOC reserves to those protocols that implement
5238  * the special handling of PFMEMALLOC skbs.
5239  */
5240 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5241 {
5242         switch (skb->protocol) {
5243         case htons(ETH_P_ARP):
5244         case htons(ETH_P_IP):
5245         case htons(ETH_P_IPV6):
5246         case htons(ETH_P_8021Q):
5247         case htons(ETH_P_8021AD):
5248                 return true;
5249         default:
5250                 return false;
5251         }
5252 }
5253
5254 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5255                              int *ret, struct net_device *orig_dev)
5256 {
5257         if (nf_hook_ingress_active(skb)) {
5258                 int ingress_retval;
5259
5260                 if (*pt_prev) {
5261                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
5262                         *pt_prev = NULL;
5263                 }
5264
5265                 rcu_read_lock();
5266                 ingress_retval = nf_hook_ingress(skb);
5267                 rcu_read_unlock();
5268                 return ingress_retval;
5269         }
5270         return 0;
5271 }
5272
5273 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
5274                                     struct packet_type **ppt_prev)
5275 {
5276         struct packet_type *ptype, *pt_prev;
5277         rx_handler_func_t *rx_handler;
5278         struct sk_buff *skb = *pskb;
5279         struct net_device *orig_dev;
5280         bool deliver_exact = false;
5281         int ret = NET_RX_DROP;
5282         __be16 type;
5283
5284         net_timestamp_check(!READ_ONCE(netdev_tstamp_prequeue), skb);
5285
5286         trace_netif_receive_skb(skb);
5287
5288         orig_dev = skb->dev;
5289
5290         skb_reset_network_header(skb);
5291         if (!skb_transport_header_was_set(skb))
5292                 skb_reset_transport_header(skb);
5293         skb_reset_mac_len(skb);
5294
5295         pt_prev = NULL;
5296
5297 another_round:
5298         skb->skb_iif = skb->dev->ifindex;
5299
5300         __this_cpu_inc(softnet_data.processed);
5301
5302         if (static_branch_unlikely(&generic_xdp_needed_key)) {
5303                 int ret2;
5304
5305                 migrate_disable();
5306                 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5307                 migrate_enable();
5308
5309                 if (ret2 != XDP_PASS) {
5310                         ret = NET_RX_DROP;
5311                         goto out;
5312                 }
5313         }
5314
5315         if (eth_type_vlan(skb->protocol)) {
5316                 skb = skb_vlan_untag(skb);
5317                 if (unlikely(!skb))
5318                         goto out;
5319         }
5320
5321         if (skb_skip_tc_classify(skb))
5322                 goto skip_classify;
5323
5324         if (pfmemalloc)
5325                 goto skip_taps;
5326
5327         list_for_each_entry_rcu(ptype, &ptype_all, list) {
5328                 if (pt_prev)
5329                         ret = deliver_skb(skb, pt_prev, orig_dev);
5330                 pt_prev = ptype;
5331         }
5332
5333         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5334                 if (pt_prev)
5335                         ret = deliver_skb(skb, pt_prev, orig_dev);
5336                 pt_prev = ptype;
5337         }
5338
5339 skip_taps:
5340 #ifdef CONFIG_NET_INGRESS
5341         if (static_branch_unlikely(&ingress_needed_key)) {
5342                 bool another = false;
5343
5344                 nf_skip_egress(skb, true);
5345                 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5346                                          &another);
5347                 if (another)
5348                         goto another_round;
5349                 if (!skb)
5350                         goto out;
5351
5352                 nf_skip_egress(skb, false);
5353                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
5354                         goto out;
5355         }
5356 #endif
5357         skb_reset_redirect(skb);
5358 skip_classify:
5359         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
5360                 goto drop;
5361
5362         if (skb_vlan_tag_present(skb)) {
5363                 if (pt_prev) {
5364                         ret = deliver_skb(skb, pt_prev, orig_dev);
5365                         pt_prev = NULL;
5366                 }
5367                 if (vlan_do_receive(&skb))
5368                         goto another_round;
5369                 else if (unlikely(!skb))
5370                         goto out;
5371         }
5372
5373         rx_handler = rcu_dereference(skb->dev->rx_handler);
5374         if (rx_handler) {
5375                 if (pt_prev) {
5376                         ret = deliver_skb(skb, pt_prev, orig_dev);
5377                         pt_prev = NULL;
5378                 }
5379                 switch (rx_handler(&skb)) {
5380                 case RX_HANDLER_CONSUMED:
5381                         ret = NET_RX_SUCCESS;
5382                         goto out;
5383                 case RX_HANDLER_ANOTHER:
5384                         goto another_round;
5385                 case RX_HANDLER_EXACT:
5386                         deliver_exact = true;
5387                         break;
5388                 case RX_HANDLER_PASS:
5389                         break;
5390                 default:
5391                         BUG();
5392                 }
5393         }
5394
5395         if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
5396 check_vlan_id:
5397                 if (skb_vlan_tag_get_id(skb)) {
5398                         /* Vlan id is non 0 and vlan_do_receive() above couldn't
5399                          * find vlan device.
5400                          */
5401                         skb->pkt_type = PACKET_OTHERHOST;
5402                 } else if (eth_type_vlan(skb->protocol)) {
5403                         /* Outer header is 802.1P with vlan 0, inner header is
5404                          * 802.1Q or 802.1AD and vlan_do_receive() above could
5405                          * not find vlan dev for vlan id 0.
5406                          */
5407                         __vlan_hwaccel_clear_tag(skb);
5408                         skb = skb_vlan_untag(skb);
5409                         if (unlikely(!skb))
5410                                 goto out;
5411                         if (vlan_do_receive(&skb))
5412                                 /* After stripping off 802.1P header with vlan 0
5413                                  * vlan dev is found for inner header.
5414                                  */
5415                                 goto another_round;
5416                         else if (unlikely(!skb))
5417                                 goto out;
5418                         else
5419                                 /* We have stripped outer 802.1P vlan 0 header.
5420                                  * But could not find vlan dev.
5421                                  * check again for vlan id to set OTHERHOST.
5422                                  */
5423                                 goto check_vlan_id;
5424                 }
5425                 /* Note: we might in the future use prio bits
5426                  * and set skb->priority like in vlan_do_receive()
5427                  * For the time being, just ignore Priority Code Point
5428                  */
5429                 __vlan_hwaccel_clear_tag(skb);
5430         }
5431
5432         type = skb->protocol;
5433
5434         /* deliver only exact match when indicated */
5435         if (likely(!deliver_exact)) {
5436                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5437                                        &ptype_base[ntohs(type) &
5438                                                    PTYPE_HASH_MASK]);
5439         }
5440
5441         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5442                                &orig_dev->ptype_specific);
5443
5444         if (unlikely(skb->dev != orig_dev)) {
5445                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5446                                        &skb->dev->ptype_specific);
5447         }
5448
5449         if (pt_prev) {
5450                 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
5451                         goto drop;
5452                 *ppt_prev = pt_prev;
5453         } else {
5454 drop:
5455                 if (!deliver_exact)
5456                         dev_core_stats_rx_dropped_inc(skb->dev);
5457                 else
5458                         dev_core_stats_rx_nohandler_inc(skb->dev);
5459                 kfree_skb_reason(skb, SKB_DROP_REASON_UNHANDLED_PROTO);
5460                 /* Jamal, now you will not able to escape explaining
5461                  * me how you were going to use this. :-)
5462                  */
5463                 ret = NET_RX_DROP;
5464         }
5465
5466 out:
5467         /* The invariant here is that if *ppt_prev is not NULL
5468          * then skb should also be non-NULL.
5469          *
5470          * Apparently *ppt_prev assignment above holds this invariant due to
5471          * skb dereferencing near it.
5472          */
5473         *pskb = skb;
5474         return ret;
5475 }
5476
5477 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5478 {
5479         struct net_device *orig_dev = skb->dev;
5480         struct packet_type *pt_prev = NULL;
5481         int ret;
5482
5483         ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5484         if (pt_prev)
5485                 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5486                                          skb->dev, pt_prev, orig_dev);
5487         return ret;
5488 }
5489
5490 /**
5491  *      netif_receive_skb_core - special purpose version of netif_receive_skb
5492  *      @skb: buffer to process
5493  *
5494  *      More direct receive version of netif_receive_skb().  It should
5495  *      only be used by callers that have a need to skip RPS and Generic XDP.
5496  *      Caller must also take care of handling if ``(page_is_)pfmemalloc``.
5497  *
5498  *      This function may only be called from softirq context and interrupts
5499  *      should be enabled.
5500  *
5501  *      Return values (usually ignored):
5502  *      NET_RX_SUCCESS: no congestion
5503  *      NET_RX_DROP: packet was dropped
5504  */
5505 int netif_receive_skb_core(struct sk_buff *skb)
5506 {
5507         int ret;
5508
5509         rcu_read_lock();
5510         ret = __netif_receive_skb_one_core(skb, false);
5511         rcu_read_unlock();
5512
5513         return ret;
5514 }
5515 EXPORT_SYMBOL(netif_receive_skb_core);
5516
5517 static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5518                                                   struct packet_type *pt_prev,
5519                                                   struct net_device *orig_dev)
5520 {
5521         struct sk_buff *skb, *next;
5522
5523         if (!pt_prev)
5524                 return;
5525         if (list_empty(head))
5526                 return;
5527         if (pt_prev->list_func != NULL)
5528                 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5529                                    ip_list_rcv, head, pt_prev, orig_dev);
5530         else
5531                 list_for_each_entry_safe(skb, next, head, list) {
5532                         skb_list_del_init(skb);
5533                         pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5534                 }
5535 }
5536
5537 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5538 {
5539         /* Fast-path assumptions:
5540          * - There is no RX handler.
5541          * - Only one packet_type matches.
5542          * If either of these fails, we will end up doing some per-packet
5543          * processing in-line, then handling the 'last ptype' for the whole
5544          * sublist.  This can't cause out-of-order delivery to any single ptype,
5545          * because the 'last ptype' must be constant across the sublist, and all
5546          * other ptypes are handled per-packet.
5547          */
5548         /* Current (common) ptype of sublist */
5549         struct packet_type *pt_curr = NULL;
5550         /* Current (common) orig_dev of sublist */
5551         struct net_device *od_curr = NULL;
5552         struct list_head sublist;
5553         struct sk_buff *skb, *next;
5554
5555         INIT_LIST_HEAD(&sublist);
5556         list_for_each_entry_safe(skb, next, head, list) {
5557                 struct net_device *orig_dev = skb->dev;
5558                 struct packet_type *pt_prev = NULL;
5559
5560                 skb_list_del_init(skb);
5561                 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5562                 if (!pt_prev)
5563                         continue;
5564                 if (pt_curr != pt_prev || od_curr != orig_dev) {
5565                         /* dispatch old sublist */
5566                         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5567                         /* start new sublist */
5568                         INIT_LIST_HEAD(&sublist);
5569                         pt_curr = pt_prev;
5570                         od_curr = orig_dev;
5571                 }
5572                 list_add_tail(&skb->list, &sublist);
5573         }
5574
5575         /* dispatch final sublist */
5576         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5577 }
5578
5579 static int __netif_receive_skb(struct sk_buff *skb)
5580 {
5581         int ret;
5582
5583         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5584                 unsigned int noreclaim_flag;
5585
5586                 /*
5587                  * PFMEMALLOC skbs are special, they should
5588                  * - be delivered to SOCK_MEMALLOC sockets only
5589                  * - stay away from userspace
5590                  * - have bounded memory usage
5591                  *
5592                  * Use PF_MEMALLOC as this saves us from propagating the allocation
5593                  * context down to all allocation sites.
5594                  */
5595                 noreclaim_flag = memalloc_noreclaim_save();
5596                 ret = __netif_receive_skb_one_core(skb, true);
5597                 memalloc_noreclaim_restore(noreclaim_flag);
5598         } else
5599                 ret = __netif_receive_skb_one_core(skb, false);
5600
5601         return ret;
5602 }
5603
5604 static void __netif_receive_skb_list(struct list_head *head)
5605 {
5606         unsigned long noreclaim_flag = 0;
5607         struct sk_buff *skb, *next;
5608         bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5609
5610         list_for_each_entry_safe(skb, next, head, list) {
5611                 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5612                         struct list_head sublist;
5613
5614                         /* Handle the previous sublist */
5615                         list_cut_before(&sublist, head, &skb->list);
5616                         if (!list_empty(&sublist))
5617                                 __netif_receive_skb_list_core(&sublist, pfmemalloc);
5618                         pfmemalloc = !pfmemalloc;
5619                         /* See comments in __netif_receive_skb */
5620                         if (pfmemalloc)
5621                                 noreclaim_flag = memalloc_noreclaim_save();
5622                         else
5623                                 memalloc_noreclaim_restore(noreclaim_flag);
5624                 }
5625         }
5626         /* Handle the remaining sublist */
5627         if (!list_empty(head))
5628                 __netif_receive_skb_list_core(head, pfmemalloc);
5629         /* Restore pflags */
5630         if (pfmemalloc)
5631                 memalloc_noreclaim_restore(noreclaim_flag);
5632 }
5633
5634 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5635 {
5636         struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5637         struct bpf_prog *new = xdp->prog;
5638         int ret = 0;
5639
5640         switch (xdp->command) {
5641         case XDP_SETUP_PROG:
5642                 rcu_assign_pointer(dev->xdp_prog, new);
5643                 if (old)
5644                         bpf_prog_put(old);
5645
5646                 if (old && !new) {
5647                         static_branch_dec(&generic_xdp_needed_key);
5648                 } else if (new && !old) {
5649                         static_branch_inc(&generic_xdp_needed_key);
5650                         dev_disable_lro(dev);
5651                         dev_disable_gro_hw(dev);
5652                 }
5653                 break;
5654
5655         default:
5656                 ret = -EINVAL;
5657                 break;
5658         }
5659
5660         return ret;
5661 }
5662
5663 static int netif_receive_skb_internal(struct sk_buff *skb)
5664 {
5665         int ret;
5666
5667         net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5668
5669         if (skb_defer_rx_timestamp(skb))
5670                 return NET_RX_SUCCESS;
5671
5672         rcu_read_lock();
5673 #ifdef CONFIG_RPS
5674         if (static_branch_unlikely(&rps_needed)) {
5675                 struct rps_dev_flow voidflow, *rflow = &voidflow;
5676                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5677
5678                 if (cpu >= 0) {
5679                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5680                         rcu_read_unlock();
5681                         return ret;
5682                 }
5683         }
5684 #endif
5685         ret = __netif_receive_skb(skb);
5686         rcu_read_unlock();
5687         return ret;
5688 }
5689
5690 void netif_receive_skb_list_internal(struct list_head *head)
5691 {
5692         struct sk_buff *skb, *next;
5693         struct list_head sublist;
5694
5695         INIT_LIST_HEAD(&sublist);
5696         list_for_each_entry_safe(skb, next, head, list) {
5697                 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5698                 skb_list_del_init(skb);
5699                 if (!skb_defer_rx_timestamp(skb))
5700                         list_add_tail(&skb->list, &sublist);
5701         }
5702         list_splice_init(&sublist, head);
5703
5704         rcu_read_lock();
5705 #ifdef CONFIG_RPS
5706         if (static_branch_unlikely(&rps_needed)) {
5707                 list_for_each_entry_safe(skb, next, head, list) {
5708                         struct rps_dev_flow voidflow, *rflow = &voidflow;
5709                         int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5710
5711                         if (cpu >= 0) {
5712                                 /* Will be handled, remove from list */
5713                                 skb_list_del_init(skb);
5714                                 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5715                         }
5716                 }
5717         }
5718 #endif
5719         __netif_receive_skb_list(head);
5720         rcu_read_unlock();
5721 }
5722
5723 /**
5724  *      netif_receive_skb - process receive buffer from network
5725  *      @skb: buffer to process
5726  *
5727  *      netif_receive_skb() is the main receive data processing function.
5728  *      It always succeeds. The buffer may be dropped during processing
5729  *      for congestion control or by the protocol layers.
5730  *
5731  *      This function may only be called from softirq context and interrupts
5732  *      should be enabled.
5733  *
5734  *      Return values (usually ignored):
5735  *      NET_RX_SUCCESS: no congestion
5736  *      NET_RX_DROP: packet was dropped
5737  */
5738 int netif_receive_skb(struct sk_buff *skb)
5739 {
5740         int ret;
5741
5742         trace_netif_receive_skb_entry(skb);
5743
5744         ret = netif_receive_skb_internal(skb);
5745         trace_netif_receive_skb_exit(ret);
5746
5747         return ret;
5748 }
5749 EXPORT_SYMBOL(netif_receive_skb);
5750
5751 /**
5752  *      netif_receive_skb_list - process many receive buffers from network
5753  *      @head: list of skbs to process.
5754  *
5755  *      Since return value of netif_receive_skb() is normally ignored, and
5756  *      wouldn't be meaningful for a list, this function returns void.
5757  *
5758  *      This function may only be called from softirq context and interrupts
5759  *      should be enabled.
5760  */
5761 void netif_receive_skb_list(struct list_head *head)
5762 {
5763         struct sk_buff *skb;
5764
5765         if (list_empty(head))
5766                 return;
5767         if (trace_netif_receive_skb_list_entry_enabled()) {
5768                 list_for_each_entry(skb, head, list)
5769                         trace_netif_receive_skb_list_entry(skb);
5770         }
5771         netif_receive_skb_list_internal(head);
5772         trace_netif_receive_skb_list_exit(0);
5773 }
5774 EXPORT_SYMBOL(netif_receive_skb_list);
5775
5776 static DEFINE_PER_CPU(struct work_struct, flush_works);
5777
5778 /* Network device is going away, flush any packets still pending */
5779 static void flush_backlog(struct work_struct *work)
5780 {
5781         struct sk_buff *skb, *tmp;
5782         struct softnet_data *sd;
5783
5784         local_bh_disable();
5785         sd = this_cpu_ptr(&softnet_data);
5786
5787         rps_lock_irq_disable(sd);
5788         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5789                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5790                         __skb_unlink(skb, &sd->input_pkt_queue);
5791                         dev_kfree_skb_irq(skb);
5792                         input_queue_head_incr(sd);
5793                 }
5794         }
5795         rps_unlock_irq_enable(sd);
5796
5797         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5798                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5799                         __skb_unlink(skb, &sd->process_queue);
5800                         kfree_skb(skb);
5801                         input_queue_head_incr(sd);
5802                 }
5803         }
5804         local_bh_enable();
5805 }
5806
5807 static bool flush_required(int cpu)
5808 {
5809 #if IS_ENABLED(CONFIG_RPS)
5810         struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5811         bool do_flush;
5812
5813         rps_lock_irq_disable(sd);
5814
5815         /* as insertion into process_queue happens with the rps lock held,
5816          * process_queue access may race only with dequeue
5817          */
5818         do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
5819                    !skb_queue_empty_lockless(&sd->process_queue);
5820         rps_unlock_irq_enable(sd);
5821
5822         return do_flush;
5823 #endif
5824         /* without RPS we can't safely check input_pkt_queue: during a
5825          * concurrent remote skb_queue_splice() we can detect as empty both
5826          * input_pkt_queue and process_queue even if the latter could end-up
5827          * containing a lot of packets.
5828          */
5829         return true;
5830 }
5831
5832 static void flush_all_backlogs(void)
5833 {
5834         static cpumask_t flush_cpus;
5835         unsigned int cpu;
5836
5837         /* since we are under rtnl lock protection we can use static data
5838          * for the cpumask and avoid allocating on stack the possibly
5839          * large mask
5840          */
5841         ASSERT_RTNL();
5842
5843         cpus_read_lock();
5844
5845         cpumask_clear(&flush_cpus);
5846         for_each_online_cpu(cpu) {
5847                 if (flush_required(cpu)) {
5848                         queue_work_on(cpu, system_highpri_wq,
5849                                       per_cpu_ptr(&flush_works, cpu));
5850                         cpumask_set_cpu(cpu, &flush_cpus);
5851                 }
5852         }
5853
5854         /* we can have in flight packet[s] on the cpus we are not flushing,
5855          * synchronize_net() in unregister_netdevice_many() will take care of
5856          * them
5857          */
5858         for_each_cpu(cpu, &flush_cpus)
5859                 flush_work(per_cpu_ptr(&flush_works, cpu));
5860
5861         cpus_read_unlock();
5862 }
5863
5864 static void net_rps_send_ipi(struct softnet_data *remsd)
5865 {
5866 #ifdef CONFIG_RPS
5867         while (remsd) {
5868                 struct softnet_data *next = remsd->rps_ipi_next;
5869
5870                 if (cpu_online(remsd->cpu))
5871                         smp_call_function_single_async(remsd->cpu, &remsd->csd);
5872                 remsd = next;
5873         }
5874 #endif
5875 }
5876
5877 /*
5878  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5879  * Note: called with local irq disabled, but exits with local irq enabled.
5880  */
5881 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5882 {
5883 #ifdef CONFIG_RPS
5884         struct softnet_data *remsd = sd->rps_ipi_list;
5885
5886         if (remsd) {
5887                 sd->rps_ipi_list = NULL;
5888
5889                 local_irq_enable();
5890
5891                 /* Send pending IPI's to kick RPS processing on remote cpus. */
5892                 net_rps_send_ipi(remsd);
5893         } else
5894 #endif
5895                 local_irq_enable();
5896 }
5897
5898 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5899 {
5900 #ifdef CONFIG_RPS
5901         return sd->rps_ipi_list != NULL;
5902 #else
5903         return false;
5904 #endif
5905 }
5906
5907 static int process_backlog(struct napi_struct *napi, int quota)
5908 {
5909         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5910         bool again = true;
5911         int work = 0;
5912
5913         /* Check if we have pending ipi, its better to send them now,
5914          * not waiting net_rx_action() end.
5915          */
5916         if (sd_has_rps_ipi_waiting(sd)) {
5917                 local_irq_disable();
5918                 net_rps_action_and_irq_enable(sd);
5919         }
5920
5921         napi->weight = READ_ONCE(dev_rx_weight);
5922         while (again) {
5923                 struct sk_buff *skb;
5924
5925                 while ((skb = __skb_dequeue(&sd->process_queue))) {
5926                         rcu_read_lock();
5927                         __netif_receive_skb(skb);
5928                         rcu_read_unlock();
5929                         input_queue_head_incr(sd);
5930                         if (++work >= quota)
5931                                 return work;
5932
5933                 }
5934
5935                 rps_lock_irq_disable(sd);
5936                 if (skb_queue_empty(&sd->input_pkt_queue)) {
5937                         /*
5938                          * Inline a custom version of __napi_complete().
5939                          * only current cpu owns and manipulates this napi,
5940                          * and NAPI_STATE_SCHED is the only possible flag set
5941                          * on backlog.
5942                          * We can use a plain write instead of clear_bit(),
5943                          * and we dont need an smp_mb() memory barrier.
5944                          */
5945                         napi->state = 0;
5946                         again = false;
5947                 } else {
5948                         skb_queue_splice_tail_init(&sd->input_pkt_queue,
5949                                                    &sd->process_queue);
5950                 }
5951                 rps_unlock_irq_enable(sd);
5952         }
5953
5954         return work;
5955 }
5956
5957 /**
5958  * __napi_schedule - schedule for receive
5959  * @n: entry to schedule
5960  *
5961  * The entry's receive function will be scheduled to run.
5962  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
5963  */
5964 void __napi_schedule(struct napi_struct *n)
5965 {
5966         unsigned long flags;
5967
5968         local_irq_save(flags);
5969         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5970         local_irq_restore(flags);
5971 }
5972 EXPORT_SYMBOL(__napi_schedule);
5973
5974 /**
5975  *      napi_schedule_prep - check if napi can be scheduled
5976  *      @n: napi context
5977  *
5978  * Test if NAPI routine is already running, and if not mark
5979  * it as running.  This is used as a condition variable to
5980  * insure only one NAPI poll instance runs.  We also make
5981  * sure there is no pending NAPI disable.
5982  */
5983 bool napi_schedule_prep(struct napi_struct *n)
5984 {
5985         unsigned long new, val = READ_ONCE(n->state);
5986
5987         do {
5988                 if (unlikely(val & NAPIF_STATE_DISABLE))
5989                         return false;
5990                 new = val | NAPIF_STATE_SCHED;
5991
5992                 /* Sets STATE_MISSED bit if STATE_SCHED was already set
5993                  * This was suggested by Alexander Duyck, as compiler
5994                  * emits better code than :
5995                  * if (val & NAPIF_STATE_SCHED)
5996                  *     new |= NAPIF_STATE_MISSED;
5997                  */
5998                 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
5999                                                    NAPIF_STATE_MISSED;
6000         } while (!try_cmpxchg(&n->state, &val, new));
6001
6002         return !(val & NAPIF_STATE_SCHED);
6003 }
6004 EXPORT_SYMBOL(napi_schedule_prep);
6005
6006 /**
6007  * __napi_schedule_irqoff - schedule for receive
6008  * @n: entry to schedule
6009  *
6010  * Variant of __napi_schedule() assuming hard irqs are masked.
6011  *
6012  * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6013  * because the interrupt disabled assumption might not be true
6014  * due to force-threaded interrupts and spinlock substitution.
6015  */
6016 void __napi_schedule_irqoff(struct napi_struct *n)
6017 {
6018         if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6019                 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6020         else
6021                 __napi_schedule(n);
6022 }
6023 EXPORT_SYMBOL(__napi_schedule_irqoff);
6024
6025 bool napi_complete_done(struct napi_struct *n, int work_done)
6026 {
6027         unsigned long flags, val, new, timeout = 0;
6028         bool ret = true;
6029
6030         /*
6031          * 1) Don't let napi dequeue from the cpu poll list
6032          *    just in case its running on a different cpu.
6033          * 2) If we are busy polling, do nothing here, we have
6034          *    the guarantee we will be called later.
6035          */
6036         if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6037                                  NAPIF_STATE_IN_BUSY_POLL)))
6038                 return false;
6039
6040         if (work_done) {
6041                 if (n->gro_bitmask)
6042                         timeout = READ_ONCE(n->dev->gro_flush_timeout);
6043                 n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
6044         }
6045         if (n->defer_hard_irqs_count > 0) {
6046                 n->defer_hard_irqs_count--;
6047                 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6048                 if (timeout)
6049                         ret = false;
6050         }
6051         if (n->gro_bitmask) {
6052                 /* When the NAPI instance uses a timeout and keeps postponing
6053                  * it, we need to bound somehow the time packets are kept in
6054                  * the GRO layer
6055                  */
6056                 napi_gro_flush(n, !!timeout);
6057         }
6058
6059         gro_normal_list(n);
6060
6061         if (unlikely(!list_empty(&n->poll_list))) {
6062                 /* If n->poll_list is not empty, we need to mask irqs */
6063                 local_irq_save(flags);
6064                 list_del_init(&n->poll_list);
6065                 local_irq_restore(flags);
6066         }
6067
6068         val = READ_ONCE(n->state);
6069         do {
6070                 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6071
6072                 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
6073                               NAPIF_STATE_SCHED_THREADED |
6074                               NAPIF_STATE_PREFER_BUSY_POLL);
6075
6076                 /* If STATE_MISSED was set, leave STATE_SCHED set,
6077                  * because we will call napi->poll() one more time.
6078                  * This C code was suggested by Alexander Duyck to help gcc.
6079                  */
6080                 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6081                                                     NAPIF_STATE_SCHED;
6082         } while (!try_cmpxchg(&n->state, &val, new));
6083
6084         if (unlikely(val & NAPIF_STATE_MISSED)) {
6085                 __napi_schedule(n);
6086                 return false;
6087         }
6088
6089         if (timeout)
6090                 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6091                               HRTIMER_MODE_REL_PINNED);
6092         return ret;
6093 }
6094 EXPORT_SYMBOL(napi_complete_done);
6095
6096 /* must be called under rcu_read_lock(), as we dont take a reference */
6097 static struct napi_struct *napi_by_id(unsigned int napi_id)
6098 {
6099         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6100         struct napi_struct *napi;
6101
6102         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6103                 if (napi->napi_id == napi_id)
6104                         return napi;
6105
6106         return NULL;
6107 }
6108
6109 #if defined(CONFIG_NET_RX_BUSY_POLL)
6110
6111 static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
6112 {
6113         if (!skip_schedule) {
6114                 gro_normal_list(napi);
6115                 __napi_schedule(napi);
6116                 return;
6117         }
6118
6119         if (napi->gro_bitmask) {
6120                 /* flush too old packets
6121                  * If HZ < 1000, flush all packets.
6122                  */
6123                 napi_gro_flush(napi, HZ >= 1000);
6124         }
6125
6126         gro_normal_list(napi);
6127         clear_bit(NAPI_STATE_SCHED, &napi->state);
6128 }
6129
6130 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
6131                            u16 budget)
6132 {
6133         bool skip_schedule = false;
6134         unsigned long timeout;
6135         int rc;
6136
6137         /* Busy polling means there is a high chance device driver hard irq
6138          * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6139          * set in napi_schedule_prep().
6140          * Since we are about to call napi->poll() once more, we can safely
6141          * clear NAPI_STATE_MISSED.
6142          *
6143          * Note: x86 could use a single "lock and ..." instruction
6144          * to perform these two clear_bit()
6145          */
6146         clear_bit(NAPI_STATE_MISSED, &napi->state);
6147         clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6148
6149         local_bh_disable();
6150
6151         if (prefer_busy_poll) {
6152                 napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
6153                 timeout = READ_ONCE(napi->dev->gro_flush_timeout);
6154                 if (napi->defer_hard_irqs_count && timeout) {
6155                         hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6156                         skip_schedule = true;
6157                 }
6158         }
6159
6160         /* All we really want here is to re-enable device interrupts.
6161          * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6162          */
6163         rc = napi->poll(napi, budget);
6164         /* We can't gro_normal_list() here, because napi->poll() might have
6165          * rearmed the napi (napi_complete_done()) in which case it could
6166          * already be running on another CPU.
6167          */
6168         trace_napi_poll(napi, rc, budget);
6169         netpoll_poll_unlock(have_poll_lock);
6170         if (rc == budget)
6171                 __busy_poll_stop(napi, skip_schedule);
6172         local_bh_enable();
6173 }
6174
6175 void napi_busy_loop(unsigned int napi_id,
6176                     bool (*loop_end)(void *, unsigned long),
6177                     void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6178 {
6179         unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6180         int (*napi_poll)(struct napi_struct *napi, int budget);
6181         void *have_poll_lock = NULL;
6182         struct napi_struct *napi;
6183
6184 restart:
6185         napi_poll = NULL;
6186
6187         rcu_read_lock();
6188
6189         napi = napi_by_id(napi_id);
6190         if (!napi)
6191                 goto out;
6192
6193         preempt_disable();
6194         for (;;) {
6195                 int work = 0;
6196
6197                 local_bh_disable();
6198                 if (!napi_poll) {
6199                         unsigned long val = READ_ONCE(napi->state);
6200
6201                         /* If multiple threads are competing for this napi,
6202                          * we avoid dirtying napi->state as much as we can.
6203                          */
6204                         if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6205                                    NAPIF_STATE_IN_BUSY_POLL)) {
6206                                 if (prefer_busy_poll)
6207                                         set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6208                                 goto count;
6209                         }
6210                         if (cmpxchg(&napi->state, val,
6211                                     val | NAPIF_STATE_IN_BUSY_POLL |
6212                                           NAPIF_STATE_SCHED) != val) {
6213                                 if (prefer_busy_poll)
6214                                         set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6215                                 goto count;
6216                         }
6217                         have_poll_lock = netpoll_poll_lock(napi);
6218                         napi_poll = napi->poll;
6219                 }
6220                 work = napi_poll(napi, budget);
6221                 trace_napi_poll(napi, work, budget);
6222                 gro_normal_list(napi);
6223 count:
6224                 if (work > 0)
6225                         __NET_ADD_STATS(dev_net(napi->dev),
6226                                         LINUX_MIB_BUSYPOLLRXPACKETS, work);
6227                 local_bh_enable();
6228
6229                 if (!loop_end || loop_end(loop_end_arg, start_time))
6230                         break;
6231
6232                 if (unlikely(need_resched())) {
6233                         if (napi_poll)
6234                                 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6235                         preempt_enable();
6236                         rcu_read_unlock();
6237                         cond_resched();
6238                         if (loop_end(loop_end_arg, start_time))
6239                                 return;
6240                         goto restart;
6241                 }
6242                 cpu_relax();
6243         }
6244         if (napi_poll)
6245                 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6246         preempt_enable();
6247 out:
6248         rcu_read_unlock();
6249 }
6250 EXPORT_SYMBOL(napi_busy_loop);
6251
6252 #endif /* CONFIG_NET_RX_BUSY_POLL */
6253
6254 static void napi_hash_add(struct napi_struct *napi)
6255 {
6256         if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
6257                 return;
6258
6259         spin_lock(&napi_hash_lock);
6260
6261         /* 0..NR_CPUS range is reserved for sender_cpu use */
6262         do {
6263                 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6264                         napi_gen_id = MIN_NAPI_ID;
6265         } while (napi_by_id(napi_gen_id));
6266         napi->napi_id = napi_gen_id;
6267
6268         hlist_add_head_rcu(&napi->napi_hash_node,
6269                            &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
6270
6271         spin_unlock(&napi_hash_lock);
6272 }
6273
6274 /* Warning : caller is responsible to make sure rcu grace period
6275  * is respected before freeing memory containing @napi
6276  */
6277 static void napi_hash_del(struct napi_struct *napi)
6278 {
6279         spin_lock(&napi_hash_lock);
6280
6281         hlist_del_init_rcu(&napi->napi_hash_node);
6282
6283         spin_unlock(&napi_hash_lock);
6284 }
6285
6286 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6287 {
6288         struct napi_struct *napi;
6289
6290         napi = container_of(timer, struct napi_struct, timer);
6291
6292         /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6293          * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6294          */
6295         if (!napi_disable_pending(napi) &&
6296             !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6297                 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6298                 __napi_schedule_irqoff(napi);
6299         }
6300
6301         return HRTIMER_NORESTART;
6302 }
6303
6304 static void init_gro_hash(struct napi_struct *napi)
6305 {
6306         int i;
6307
6308         for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6309                 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6310                 napi->gro_hash[i].count = 0;
6311         }
6312         napi->gro_bitmask = 0;
6313 }
6314
6315 int dev_set_threaded(struct net_device *dev, bool threaded)
6316 {
6317         struct napi_struct *napi;
6318         int err = 0;
6319
6320         if (dev->threaded == threaded)
6321                 return 0;
6322
6323         if (threaded) {
6324                 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6325                         if (!napi->thread) {
6326                                 err = napi_kthread_create(napi);
6327                                 if (err) {
6328                                         threaded = false;
6329                                         break;
6330                                 }
6331                         }
6332                 }
6333         }
6334
6335         dev->threaded = threaded;
6336
6337         /* Make sure kthread is created before THREADED bit
6338          * is set.
6339          */
6340         smp_mb__before_atomic();
6341
6342         /* Setting/unsetting threaded mode on a napi might not immediately
6343          * take effect, if the current napi instance is actively being
6344          * polled. In this case, the switch between threaded mode and
6345          * softirq mode will happen in the next round of napi_schedule().
6346          * This should not cause hiccups/stalls to the live traffic.
6347          */
6348         list_for_each_entry(napi, &dev->napi_list, dev_list) {
6349                 if (threaded)
6350                         set_bit(NAPI_STATE_THREADED, &napi->state);
6351                 else
6352                         clear_bit(NAPI_STATE_THREADED, &napi->state);
6353         }
6354
6355         return err;
6356 }
6357 EXPORT_SYMBOL(dev_set_threaded);
6358
6359 void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
6360                            int (*poll)(struct napi_struct *, int), int weight)
6361 {
6362         if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6363                 return;
6364
6365         INIT_LIST_HEAD(&napi->poll_list);
6366         INIT_HLIST_NODE(&napi->napi_hash_node);
6367         hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6368         napi->timer.function = napi_watchdog;
6369         init_gro_hash(napi);
6370         napi->skb = NULL;
6371         INIT_LIST_HEAD(&napi->rx_list);
6372         napi->rx_count = 0;
6373         napi->poll = poll;
6374         if (weight > NAPI_POLL_WEIGHT)
6375                 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6376                                 weight);
6377         napi->weight = weight;
6378         napi->dev = dev;
6379 #ifdef CONFIG_NETPOLL
6380         napi->poll_owner = -1;
6381 #endif
6382         set_bit(NAPI_STATE_SCHED, &napi->state);
6383         set_bit(NAPI_STATE_NPSVC, &napi->state);
6384         list_add_rcu(&napi->dev_list, &dev->napi_list);
6385         napi_hash_add(napi);
6386         napi_get_frags_check(napi);
6387         /* Create kthread for this napi if dev->threaded is set.
6388          * Clear dev->threaded if kthread creation failed so that
6389          * threaded mode will not be enabled in napi_enable().
6390          */
6391         if (dev->threaded && napi_kthread_create(napi))
6392                 dev->threaded = 0;
6393 }
6394 EXPORT_SYMBOL(netif_napi_add_weight);
6395
6396 void napi_disable(struct napi_struct *n)
6397 {
6398         unsigned long val, new;
6399
6400         might_sleep();
6401         set_bit(NAPI_STATE_DISABLE, &n->state);
6402
6403         val = READ_ONCE(n->state);
6404         do {
6405                 while (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) {
6406                         usleep_range(20, 200);
6407                         val = READ_ONCE(n->state);
6408                 }
6409
6410                 new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC;
6411                 new &= ~(NAPIF_STATE_THREADED | NAPIF_STATE_PREFER_BUSY_POLL);
6412         } while (!try_cmpxchg(&n->state, &val, new));
6413
6414         hrtimer_cancel(&n->timer);
6415
6416         clear_bit(NAPI_STATE_DISABLE, &n->state);
6417 }
6418 EXPORT_SYMBOL(napi_disable);
6419
6420 /**
6421  *      napi_enable - enable NAPI scheduling
6422  *      @n: NAPI context
6423  *
6424  * Resume NAPI from being scheduled on this context.
6425  * Must be paired with napi_disable.
6426  */
6427 void napi_enable(struct napi_struct *n)
6428 {
6429         unsigned long new, val = READ_ONCE(n->state);
6430
6431         do {
6432                 BUG_ON(!test_bit(NAPI_STATE_SCHED, &val));
6433
6434                 new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC);
6435                 if (n->dev->threaded && n->thread)
6436                         new |= NAPIF_STATE_THREADED;
6437         } while (!try_cmpxchg(&n->state, &val, new));
6438 }
6439 EXPORT_SYMBOL(napi_enable);
6440
6441 static void flush_gro_hash(struct napi_struct *napi)
6442 {
6443         int i;
6444
6445         for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6446                 struct sk_buff *skb, *n;
6447
6448                 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
6449                         kfree_skb(skb);
6450                 napi->gro_hash[i].count = 0;
6451         }
6452 }
6453
6454 /* Must be called in process context */
6455 void __netif_napi_del(struct napi_struct *napi)
6456 {
6457         if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
6458                 return;
6459
6460         napi_hash_del(napi);
6461         list_del_rcu(&napi->dev_list);
6462         napi_free_frags(napi);
6463
6464         flush_gro_hash(napi);
6465         napi->gro_bitmask = 0;
6466
6467         if (napi->thread) {
6468                 kthread_stop(napi->thread);
6469                 napi->thread = NULL;
6470         }
6471 }
6472 EXPORT_SYMBOL(__netif_napi_del);
6473
6474 static int __napi_poll(struct napi_struct *n, bool *repoll)
6475 {
6476         int work, weight;
6477
6478         weight = n->weight;
6479
6480         /* This NAPI_STATE_SCHED test is for avoiding a race
6481          * with netpoll's poll_napi().  Only the entity which
6482          * obtains the lock and sees NAPI_STATE_SCHED set will
6483          * actually make the ->poll() call.  Therefore we avoid
6484          * accidentally calling ->poll() when NAPI is not scheduled.
6485          */
6486         work = 0;
6487         if (test_bit(NAPI_STATE_SCHED, &n->state)) {
6488                 work = n->poll(n, weight);
6489                 trace_napi_poll(n, work, weight);
6490         }
6491
6492         if (unlikely(work > weight))
6493                 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
6494                                 n->poll, work, weight);
6495
6496         if (likely(work < weight))
6497                 return work;
6498
6499         /* Drivers must not modify the NAPI state if they
6500          * consume the entire weight.  In such cases this code
6501          * still "owns" the NAPI instance and therefore can
6502          * move the instance around on the list at-will.
6503          */
6504         if (unlikely(napi_disable_pending(n))) {
6505                 napi_complete(n);
6506                 return work;
6507         }
6508
6509         /* The NAPI context has more processing work, but busy-polling
6510          * is preferred. Exit early.
6511          */
6512         if (napi_prefer_busy_poll(n)) {
6513                 if (napi_complete_done(n, work)) {
6514                         /* If timeout is not set, we need to make sure
6515                          * that the NAPI is re-scheduled.
6516                          */
6517                         napi_schedule(n);
6518                 }
6519                 return work;
6520         }
6521
6522         if (n->gro_bitmask) {
6523                 /* flush too old packets
6524                  * If HZ < 1000, flush all packets.
6525                  */
6526                 napi_gro_flush(n, HZ >= 1000);
6527         }
6528
6529         gro_normal_list(n);
6530
6531         /* Some drivers may have called napi_schedule
6532          * prior to exhausting their budget.
6533          */
6534         if (unlikely(!list_empty(&n->poll_list))) {
6535                 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6536                              n->dev ? n->dev->name : "backlog");
6537                 return work;
6538         }
6539
6540         *repoll = true;
6541
6542         return work;
6543 }
6544
6545 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
6546 {
6547         bool do_repoll = false;
6548         void *have;
6549         int work;
6550
6551         list_del_init(&n->poll_list);
6552
6553         have = netpoll_poll_lock(n);
6554
6555         work = __napi_poll(n, &do_repoll);
6556
6557         if (do_repoll)
6558                 list_add_tail(&n->poll_list, repoll);
6559
6560         netpoll_poll_unlock(have);
6561
6562         return work;
6563 }
6564
6565 static int napi_thread_wait(struct napi_struct *napi)
6566 {
6567         bool woken = false;
6568
6569         set_current_state(TASK_INTERRUPTIBLE);
6570
6571         while (!kthread_should_stop()) {
6572                 /* Testing SCHED_THREADED bit here to make sure the current
6573                  * kthread owns this napi and could poll on this napi.
6574                  * Testing SCHED bit is not enough because SCHED bit might be
6575                  * set by some other busy poll thread or by napi_disable().
6576                  */
6577                 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) {
6578                         WARN_ON(!list_empty(&napi->poll_list));
6579                         __set_current_state(TASK_RUNNING);
6580                         return 0;
6581                 }
6582
6583                 schedule();
6584                 /* woken being true indicates this thread owns this napi. */
6585                 woken = true;
6586                 set_current_state(TASK_INTERRUPTIBLE);
6587         }
6588         __set_current_state(TASK_RUNNING);
6589
6590         return -1;
6591 }
6592
6593 static int napi_threaded_poll(void *data)
6594 {
6595         struct napi_struct *napi = data;
6596         void *have;
6597
6598         while (!napi_thread_wait(napi)) {
6599                 for (;;) {
6600                         bool repoll = false;
6601
6602                         local_bh_disable();
6603
6604                         have = netpoll_poll_lock(napi);
6605                         __napi_poll(napi, &repoll);
6606                         netpoll_poll_unlock(have);
6607
6608                         local_bh_enable();
6609
6610                         if (!repoll)
6611                                 break;
6612
6613                         cond_resched();
6614                 }
6615         }
6616         return 0;
6617 }
6618
6619 static void skb_defer_free_flush(struct softnet_data *sd)
6620 {
6621         struct sk_buff *skb, *next;
6622
6623         /* Paired with WRITE_ONCE() in skb_attempt_defer_free() */
6624         if (!READ_ONCE(sd->defer_list))
6625                 return;
6626
6627         spin_lock_irq(&sd->defer_lock);
6628         skb = sd->defer_list;
6629         sd->defer_list = NULL;
6630         sd->defer_count = 0;
6631         spin_unlock_irq(&sd->defer_lock);
6632
6633         while (skb != NULL) {
6634                 next = skb->next;
6635                 napi_consume_skb(skb, 1);
6636                 skb = next;
6637         }
6638 }
6639
6640 static __latent_entropy void net_rx_action(struct softirq_action *h)
6641 {
6642         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6643         unsigned long time_limit = jiffies +
6644                 usecs_to_jiffies(READ_ONCE(netdev_budget_usecs));
6645         int budget = READ_ONCE(netdev_budget);
6646         LIST_HEAD(list);
6647         LIST_HEAD(repoll);
6648
6649         sd->in_net_rx_action = true;
6650         local_irq_disable();
6651         list_splice_init(&sd->poll_list, &list);
6652         local_irq_enable();
6653
6654         for (;;) {
6655                 struct napi_struct *n;
6656
6657                 skb_defer_free_flush(sd);
6658
6659                 if (list_empty(&list)) {
6660                         sd->in_net_rx_action = false;
6661                         if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
6662                                 goto end;
6663                         break;
6664                 }
6665
6666                 n = list_first_entry(&list, struct napi_struct, poll_list);
6667                 budget -= napi_poll(n, &repoll);
6668
6669                 /* If softirq window is exhausted then punt.
6670                  * Allow this to run for 2 jiffies since which will allow
6671                  * an average latency of 1.5/HZ.
6672                  */
6673                 if (unlikely(budget <= 0 ||
6674                              time_after_eq(jiffies, time_limit))) {
6675                         sd->time_squeeze++;
6676                         break;
6677                 }
6678         }
6679
6680         local_irq_disable();
6681
6682         list_splice_tail_init(&sd->poll_list, &list);
6683         list_splice_tail(&repoll, &list);
6684         list_splice(&list, &sd->poll_list);
6685         if (!list_empty(&sd->poll_list))
6686                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
6687         else
6688                 sd->in_net_rx_action = false;
6689
6690         net_rps_action_and_irq_enable(sd);
6691 end:;
6692 }
6693
6694 struct netdev_adjacent {
6695         struct net_device *dev;
6696         netdevice_tracker dev_tracker;
6697
6698         /* upper master flag, there can only be one master device per list */
6699         bool master;
6700
6701         /* lookup ignore flag */
6702         bool ignore;
6703
6704         /* counter for the number of times this device was added to us */
6705         u16 ref_nr;
6706
6707         /* private field for the users */
6708         void *private;
6709
6710         struct list_head list;
6711         struct rcu_head rcu;
6712 };
6713
6714 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
6715                                                  struct list_head *adj_list)
6716 {
6717         struct netdev_adjacent *adj;
6718
6719         list_for_each_entry(adj, adj_list, list) {
6720                 if (adj->dev == adj_dev)
6721                         return adj;
6722         }
6723         return NULL;
6724 }
6725
6726 static int ____netdev_has_upper_dev(struct net_device *upper_dev,
6727                                     struct netdev_nested_priv *priv)
6728 {
6729         struct net_device *dev = (struct net_device *)priv->data;
6730
6731         return upper_dev == dev;
6732 }
6733
6734 /**
6735  * netdev_has_upper_dev - Check if device is linked to an upper device
6736  * @dev: device
6737  * @upper_dev: upper device to check
6738  *
6739  * Find out if a device is linked to specified upper device and return true
6740  * in case it is. Note that this checks only immediate upper device,
6741  * not through a complete stack of devices. The caller must hold the RTNL lock.
6742  */
6743 bool netdev_has_upper_dev(struct net_device *dev,
6744                           struct net_device *upper_dev)
6745 {
6746         struct netdev_nested_priv priv = {
6747                 .data = (void *)upper_dev,
6748         };
6749
6750         ASSERT_RTNL();
6751
6752         return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
6753                                              &priv);
6754 }
6755 EXPORT_SYMBOL(netdev_has_upper_dev);
6756
6757 /**
6758  * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
6759  * @dev: device
6760  * @upper_dev: upper device to check
6761  *
6762  * Find out if a device is linked to specified upper device and return true
6763  * in case it is. Note that this checks the entire upper device chain.
6764  * The caller must hold rcu lock.
6765  */
6766
6767 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
6768                                   struct net_device *upper_dev)
6769 {
6770         struct netdev_nested_priv priv = {
6771                 .data = (void *)upper_dev,
6772         };
6773
6774         return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
6775                                                &priv);
6776 }
6777 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
6778
6779 /**
6780  * netdev_has_any_upper_dev - Check if device is linked to some device
6781  * @dev: device
6782  *
6783  * Find out if a device is linked to an upper device and return true in case
6784  * it is. The caller must hold the RTNL lock.
6785  */
6786 bool netdev_has_any_upper_dev(struct net_device *dev)
6787 {
6788         ASSERT_RTNL();
6789
6790         return !list_empty(&dev->adj_list.upper);
6791 }
6792 EXPORT_SYMBOL(netdev_has_any_upper_dev);
6793
6794 /**
6795  * netdev_master_upper_dev_get - Get master upper device
6796  * @dev: device
6797  *
6798  * Find a master upper device and return pointer to it or NULL in case
6799  * it's not there. The caller must hold the RTNL lock.
6800  */
6801 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
6802 {
6803         struct netdev_adjacent *upper;
6804
6805         ASSERT_RTNL();
6806
6807         if (list_empty(&dev->adj_list.upper))
6808                 return NULL;
6809
6810         upper = list_first_entry(&dev->adj_list.upper,
6811                                  struct netdev_adjacent, list);
6812         if (likely(upper->master))
6813                 return upper->dev;
6814         return NULL;
6815 }
6816 EXPORT_SYMBOL(netdev_master_upper_dev_get);
6817
6818 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
6819 {
6820         struct netdev_adjacent *upper;
6821
6822         ASSERT_RTNL();
6823
6824         if (list_empty(&dev->adj_list.upper))
6825                 return NULL;
6826
6827         upper = list_first_entry(&dev->adj_list.upper,
6828                                  struct netdev_adjacent, list);
6829         if (likely(upper->master) && !upper->ignore)
6830                 return upper->dev;
6831         return NULL;
6832 }
6833
6834 /**
6835  * netdev_has_any_lower_dev - Check if device is linked to some device
6836  * @dev: device
6837  *
6838  * Find out if a device is linked to a lower device and return true in case
6839  * it is. The caller must hold the RTNL lock.
6840  */
6841 static bool netdev_has_any_lower_dev(struct net_device *dev)
6842 {
6843         ASSERT_RTNL();
6844
6845         return !list_empty(&dev->adj_list.lower);
6846 }
6847
6848 void *netdev_adjacent_get_private(struct list_head *adj_list)
6849 {
6850         struct netdev_adjacent *adj;
6851
6852         adj = list_entry(adj_list, struct netdev_adjacent, list);
6853
6854         return adj->private;
6855 }
6856 EXPORT_SYMBOL(netdev_adjacent_get_private);
6857
6858 /**
6859  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
6860  * @dev: device
6861  * @iter: list_head ** of the current position
6862  *
6863  * Gets the next device from the dev's upper list, starting from iter
6864  * position. The caller must hold RCU read lock.
6865  */
6866 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
6867                                                  struct list_head **iter)
6868 {
6869         struct netdev_adjacent *upper;
6870
6871         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6872
6873         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6874
6875         if (&upper->list == &dev->adj_list.upper)
6876                 return NULL;
6877
6878         *iter = &upper->list;
6879
6880         return upper->dev;
6881 }
6882 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
6883
6884 static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
6885                                                   struct list_head **iter,
6886                                                   bool *ignore)
6887 {
6888         struct netdev_adjacent *upper;
6889
6890         upper = list_entry((*iter)->next, struct netdev_adjacent, list);
6891
6892         if (&upper->list == &dev->adj_list.upper)
6893                 return NULL;
6894
6895         *iter = &upper->list;
6896         *ignore = upper->ignore;
6897
6898         return upper->dev;
6899 }
6900
6901 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
6902                                                     struct list_head **iter)
6903 {
6904         struct netdev_adjacent *upper;
6905
6906         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6907
6908         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6909
6910         if (&upper->list == &dev->adj_list.upper)
6911                 return NULL;
6912
6913         *iter = &upper->list;
6914
6915         return upper->dev;
6916 }
6917
6918 static int __netdev_walk_all_upper_dev(struct net_device *dev,
6919                                        int (*fn)(struct net_device *dev,
6920                                          struct netdev_nested_priv *priv),
6921                                        struct netdev_nested_priv *priv)
6922 {
6923         struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
6924         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
6925         int ret, cur = 0;
6926         bool ignore;
6927
6928         now = dev;
6929         iter = &dev->adj_list.upper;
6930
6931         while (1) {
6932                 if (now != dev) {
6933                         ret = fn(now, priv);
6934                         if (ret)
6935                                 return ret;
6936                 }
6937
6938                 next = NULL;
6939                 while (1) {
6940                         udev = __netdev_next_upper_dev(now, &iter, &ignore);
6941                         if (!udev)
6942                                 break;
6943                         if (ignore)
6944                                 continue;
6945
6946                         next = udev;
6947                         niter = &udev->adj_list.upper;
6948                         dev_stack[cur] = now;
6949                         iter_stack[cur++] = iter;
6950                         break;
6951                 }
6952
6953                 if (!next) {
6954                         if (!cur)
6955                                 return 0;
6956                         next = dev_stack[--cur];
6957                         niter = iter_stack[cur];
6958                 }
6959
6960                 now = next;
6961                 iter = niter;
6962         }
6963
6964         return 0;
6965 }
6966
6967 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
6968                                   int (*fn)(struct net_device *dev,
6969                                             struct netdev_nested_priv *priv),
6970                                   struct netdev_nested_priv *priv)
6971 {
6972         struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
6973         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
6974         int ret, cur = 0;
6975
6976         now = dev;
6977         iter = &dev->adj_list.upper;
6978
6979         while (1) {
6980                 if (now != dev) {
6981                         ret = fn(now, priv);
6982                         if (ret)
6983                                 return ret;
6984                 }
6985
6986                 next = NULL;
6987                 while (1) {
6988                         udev = netdev_next_upper_dev_rcu(now, &iter);
6989                         if (!udev)
6990                                 break;
6991
6992                         next = udev;
6993                         niter = &udev->adj_list.upper;
6994                         dev_stack[cur] = now;
6995                         iter_stack[cur++] = iter;
6996                         break;
6997                 }
6998
6999                 if (!next) {
7000                         if (!cur)
7001                                 return 0;
7002                         next = dev_stack[--cur];
7003                         niter = iter_stack[cur];
7004                 }
7005
7006                 now = next;
7007                 iter = niter;
7008         }
7009
7010         return 0;
7011 }
7012 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
7013
7014 static bool __netdev_has_upper_dev(struct net_device *dev,
7015                                    struct net_device *upper_dev)
7016 {
7017         struct netdev_nested_priv priv = {
7018                 .flags = 0,
7019                 .data = (void *)upper_dev,
7020         };
7021
7022         ASSERT_RTNL();
7023
7024         return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
7025                                            &priv);
7026 }
7027
7028 /**
7029  * netdev_lower_get_next_private - Get the next ->private from the
7030  *                                 lower neighbour list
7031  * @dev: device
7032  * @iter: list_head ** of the current position
7033  *
7034  * Gets the next netdev_adjacent->private from the dev's lower neighbour
7035  * list, starting from iter position. The caller must hold either hold the
7036  * RTNL lock or its own locking that guarantees that the neighbour lower
7037  * list will remain unchanged.
7038  */
7039 void *netdev_lower_get_next_private(struct net_device *dev,
7040                                     struct list_head **iter)
7041 {
7042         struct netdev_adjacent *lower;
7043
7044         lower = list_entry(*iter, struct netdev_adjacent, list);
7045
7046         if (&lower->list == &dev->adj_list.lower)
7047                 return NULL;
7048
7049         *iter = lower->list.next;
7050
7051         return lower->private;
7052 }
7053 EXPORT_SYMBOL(netdev_lower_get_next_private);
7054
7055 /**
7056  * netdev_lower_get_next_private_rcu - Get the next ->private from the
7057  *                                     lower neighbour list, RCU
7058  *                                     variant
7059  * @dev: device
7060  * @iter: list_head ** of the current position
7061  *
7062  * Gets the next netdev_adjacent->private from the dev's lower neighbour
7063  * list, starting from iter position. The caller must hold RCU read lock.
7064  */
7065 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
7066                                         struct list_head **iter)
7067 {
7068         struct netdev_adjacent *lower;
7069
7070         WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
7071
7072         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7073
7074         if (&lower->list == &dev->adj_list.lower)
7075                 return NULL;
7076
7077         *iter = &lower->list;
7078
7079         return lower->private;
7080 }
7081 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
7082
7083 /**
7084  * netdev_lower_get_next - Get the next device from the lower neighbour
7085  *                         list
7086  * @dev: device
7087  * @iter: list_head ** of the current position
7088  *
7089  * Gets the next netdev_adjacent from the dev's lower neighbour
7090  * list, starting from iter position. The caller must hold RTNL lock or
7091  * its own locking that guarantees that the neighbour lower
7092  * list will remain unchanged.
7093  */
7094 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
7095 {
7096         struct netdev_adjacent *lower;
7097
7098         lower = list_entry(*iter, struct netdev_adjacent, list);
7099
7100         if (&lower->list == &dev->adj_list.lower)
7101                 return NULL;
7102
7103         *iter = lower->list.next;
7104
7105         return lower->dev;
7106 }
7107 EXPORT_SYMBOL(netdev_lower_get_next);
7108
7109 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
7110                                                 struct list_head **iter)
7111 {
7112         struct netdev_adjacent *lower;
7113
7114         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7115
7116         if (&lower->list == &dev->adj_list.lower)
7117                 return NULL;
7118
7119         *iter = &lower->list;
7120
7121         return lower->dev;
7122 }
7123
7124 static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
7125                                                   struct list_head **iter,
7126                                                   bool *ignore)
7127 {
7128         struct netdev_adjacent *lower;
7129
7130         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7131
7132         if (&lower->list == &dev->adj_list.lower)
7133                 return NULL;
7134
7135         *iter = &lower->list;
7136         *ignore = lower->ignore;
7137
7138         return lower->dev;
7139 }
7140
7141 int netdev_walk_all_lower_dev(struct net_device *dev,
7142                               int (*fn)(struct net_device *dev,
7143                                         struct netdev_nested_priv *priv),
7144                               struct netdev_nested_priv *priv)
7145 {
7146         struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7147         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7148         int ret, cur = 0;
7149
7150         now = dev;
7151         iter = &dev->adj_list.lower;
7152
7153         while (1) {
7154                 if (now != dev) {
7155                         ret = fn(now, priv);
7156                         if (ret)
7157                                 return ret;
7158                 }
7159
7160                 next = NULL;
7161                 while (1) {
7162                         ldev = netdev_next_lower_dev(now, &iter);
7163                         if (!ldev)
7164                                 break;
7165
7166                         next = ldev;
7167                         niter = &ldev->adj_list.lower;
7168                         dev_stack[cur] = now;
7169                         iter_stack[cur++] = iter;
7170                         break;
7171                 }
7172
7173                 if (!next) {
7174                         if (!cur)
7175                                 return 0;
7176                         next = dev_stack[--cur];
7177                         niter = iter_stack[cur];
7178                 }
7179
7180                 now = next;
7181                 iter = niter;
7182         }
7183
7184         return 0;
7185 }
7186 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
7187
7188 static int __netdev_walk_all_lower_dev(struct net_device *dev,
7189                                        int (*fn)(struct net_device *dev,
7190                                          struct netdev_nested_priv *priv),
7191                                        struct netdev_nested_priv *priv)
7192 {
7193         struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7194         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7195         int ret, cur = 0;
7196         bool ignore;
7197
7198         now = dev;
7199         iter = &dev->adj_list.lower;
7200
7201         while (1) {
7202                 if (now != dev) {
7203                         ret = fn(now, priv);
7204                         if (ret)
7205                                 return ret;
7206                 }
7207
7208                 next = NULL;
7209                 while (1) {
7210                         ldev = __netdev_next_lower_dev(now, &iter, &ignore);
7211                         if (!ldev)
7212                                 break;
7213                         if (ignore)
7214                                 continue;
7215
7216                         next = ldev;
7217                         niter = &ldev->adj_list.lower;
7218                         dev_stack[cur] = now;
7219                         iter_stack[cur++] = iter;
7220                         break;
7221                 }
7222
7223                 if (!next) {
7224                         if (!cur)
7225                                 return 0;
7226                         next = dev_stack[--cur];
7227                         niter = iter_stack[cur];
7228                 }
7229
7230                 now = next;
7231                 iter = niter;
7232         }
7233
7234         return 0;
7235 }
7236
7237 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
7238                                              struct list_head **iter)
7239 {
7240         struct netdev_adjacent *lower;
7241
7242         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7243         if (&lower->list == &dev->adj_list.lower)
7244                 return NULL;
7245
7246         *iter = &lower->list;
7247
7248         return lower->dev;
7249 }
7250 EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
7251
7252 static u8 __netdev_upper_depth(struct net_device *dev)
7253 {
7254         struct net_device *udev;
7255         struct list_head *iter;
7256         u8 max_depth = 0;
7257         bool ignore;
7258
7259         for (iter = &dev->adj_list.upper,
7260              udev = __netdev_next_upper_dev(dev, &iter, &ignore);
7261              udev;
7262              udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
7263                 if (ignore)
7264                         continue;
7265                 if (max_depth < udev->upper_level)
7266                         max_depth = udev->upper_level;
7267         }
7268
7269         return max_depth;
7270 }
7271
7272 static u8 __netdev_lower_depth(struct net_device *dev)
7273 {
7274         struct net_device *ldev;
7275         struct list_head *iter;
7276         u8 max_depth = 0;
7277         bool ignore;
7278
7279         for (iter = &dev->adj_list.lower,
7280              ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
7281              ldev;
7282              ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
7283                 if (ignore)
7284                         continue;
7285                 if (max_depth < ldev->lower_level)
7286                         max_depth = ldev->lower_level;
7287         }
7288
7289         return max_depth;
7290 }
7291
7292 static int __netdev_update_upper_level(struct net_device *dev,
7293                                        struct netdev_nested_priv *__unused)
7294 {
7295         dev->upper_level = __netdev_upper_depth(dev) + 1;
7296         return 0;
7297 }
7298
7299 #ifdef CONFIG_LOCKDEP
7300 static LIST_HEAD(net_unlink_list);
7301
7302 static void net_unlink_todo(struct net_device *dev)
7303 {
7304         if (list_empty(&dev->unlink_list))
7305                 list_add_tail(&dev->unlink_list, &net_unlink_list);
7306 }
7307 #endif
7308
7309 static int __netdev_update_lower_level(struct net_device *dev,
7310                                        struct netdev_nested_priv *priv)
7311 {
7312         dev->lower_level = __netdev_lower_depth(dev) + 1;
7313
7314 #ifdef CONFIG_LOCKDEP
7315         if (!priv)
7316                 return 0;
7317
7318         if (priv->flags & NESTED_SYNC_IMM)
7319                 dev->nested_level = dev->lower_level - 1;
7320         if (priv->flags & NESTED_SYNC_TODO)
7321                 net_unlink_todo(dev);
7322 #endif
7323         return 0;
7324 }
7325
7326 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
7327                                   int (*fn)(struct net_device *dev,
7328                                             struct netdev_nested_priv *priv),
7329                                   struct netdev_nested_priv *priv)
7330 {
7331         struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7332         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7333         int ret, cur = 0;
7334
7335         now = dev;
7336         iter = &dev->adj_list.lower;
7337
7338         while (1) {
7339                 if (now != dev) {
7340                         ret = fn(now, priv);
7341                         if (ret)
7342                                 return ret;
7343                 }
7344
7345                 next = NULL;
7346                 while (1) {
7347                         ldev = netdev_next_lower_dev_rcu(now, &iter);
7348                         if (!ldev)
7349                                 break;
7350
7351                         next = ldev;
7352                         niter = &ldev->adj_list.lower;
7353                         dev_stack[cur] = now;
7354                         iter_stack[cur++] = iter;
7355                         break;
7356                 }
7357
7358                 if (!next) {
7359                         if (!cur)
7360                                 return 0;
7361                         next = dev_stack[--cur];
7362                         niter = iter_stack[cur];
7363                 }
7364
7365                 now = next;
7366                 iter = niter;
7367         }
7368
7369         return 0;
7370 }
7371 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
7372
7373 /**
7374  * netdev_lower_get_first_private_rcu - Get the first ->private from the
7375  *                                     lower neighbour list, RCU
7376  *                                     variant
7377  * @dev: device
7378  *
7379  * Gets the first netdev_adjacent->private from the dev's lower neighbour
7380  * list. The caller must hold RCU read lock.
7381  */
7382 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
7383 {
7384         struct netdev_adjacent *lower;
7385
7386         lower = list_first_or_null_rcu(&dev->adj_list.lower,
7387                         struct netdev_adjacent, list);
7388         if (lower)
7389                 return lower->private;
7390         return NULL;
7391 }
7392 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
7393
7394 /**
7395  * netdev_master_upper_dev_get_rcu - Get master upper device
7396  * @dev: device
7397  *
7398  * Find a master upper device and return pointer to it or NULL in case
7399  * it's not there. The caller must hold the RCU read lock.
7400  */
7401 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
7402 {
7403         struct netdev_adjacent *upper;
7404
7405         upper = list_first_or_null_rcu(&dev->adj_list.upper,
7406                                        struct netdev_adjacent, list);
7407         if (upper && likely(upper->master))
7408                 return upper->dev;
7409         return NULL;
7410 }
7411 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
7412
7413 static int netdev_adjacent_sysfs_add(struct net_device *dev,
7414                               struct net_device *adj_dev,
7415                               struct list_head *dev_list)
7416 {
7417         char linkname[IFNAMSIZ+7];
7418
7419         sprintf(linkname, dev_list == &dev->adj_list.upper ?
7420                 "upper_%s" : "lower_%s", adj_dev->name);
7421         return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
7422                                  linkname);
7423 }
7424 static void netdev_adjacent_sysfs_del(struct net_device *dev,
7425                                char *name,
7426                                struct list_head *dev_list)
7427 {
7428         char linkname[IFNAMSIZ+7];
7429
7430         sprintf(linkname, dev_list == &dev->adj_list.upper ?
7431                 "upper_%s" : "lower_%s", name);
7432         sysfs_remove_link(&(dev->dev.kobj), linkname);
7433 }
7434
7435 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
7436                                                  struct net_device *adj_dev,
7437                                                  struct list_head *dev_list)
7438 {
7439         return (dev_list == &dev->adj_list.upper ||
7440                 dev_list == &dev->adj_list.lower) &&
7441                 net_eq(dev_net(dev), dev_net(adj_dev));
7442 }
7443
7444 static int __netdev_adjacent_dev_insert(struct net_device *dev,
7445                                         struct net_device *adj_dev,
7446                                         struct list_head *dev_list,
7447                                         void *private, bool master)
7448 {
7449         struct netdev_adjacent *adj;
7450         int ret;
7451
7452         adj = __netdev_find_adj(adj_dev, dev_list);
7453
7454         if (adj) {
7455                 adj->ref_nr += 1;
7456                 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
7457                          dev->name, adj_dev->name, adj->ref_nr);
7458
7459                 return 0;
7460         }
7461
7462         adj = kmalloc(sizeof(*adj), GFP_KERNEL);
7463         if (!adj)
7464                 return -ENOMEM;
7465
7466         adj->dev = adj_dev;
7467         adj->master = master;
7468         adj->ref_nr = 1;
7469         adj->private = private;
7470         adj->ignore = false;
7471         netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL);
7472
7473         pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
7474                  dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
7475
7476         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
7477                 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
7478                 if (ret)
7479                         goto free_adj;
7480         }
7481
7482         /* Ensure that master link is always the first item in list. */
7483         if (master) {
7484                 ret = sysfs_create_link(&(dev->dev.kobj),
7485                                         &(adj_dev->dev.kobj), "master");
7486                 if (ret)
7487                         goto remove_symlinks;
7488
7489                 list_add_rcu(&adj->list, dev_list);
7490         } else {
7491                 list_add_tail_rcu(&adj->list, dev_list);
7492         }
7493
7494         return 0;
7495
7496 remove_symlinks:
7497         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7498                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7499 free_adj:
7500         netdev_put(adj_dev, &adj->dev_tracker);
7501         kfree(adj);
7502
7503         return ret;
7504 }
7505
7506 static void __netdev_adjacent_dev_remove(struct net_device *dev,
7507                                          struct net_device *adj_dev,
7508                                          u16 ref_nr,
7509                                          struct list_head *dev_list)
7510 {
7511         struct netdev_adjacent *adj;
7512
7513         pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
7514                  dev->name, adj_dev->name, ref_nr);
7515
7516         adj = __netdev_find_adj(adj_dev, dev_list);
7517
7518         if (!adj) {
7519                 pr_err("Adjacency does not exist for device %s from %s\n",
7520                        dev->name, adj_dev->name);
7521                 WARN_ON(1);
7522                 return;
7523         }
7524
7525         if (adj->ref_nr > ref_nr) {
7526                 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
7527                          dev->name, adj_dev->name, ref_nr,
7528                          adj->ref_nr - ref_nr);
7529                 adj->ref_nr -= ref_nr;
7530                 return;
7531         }
7532
7533         if (adj->master)
7534                 sysfs_remove_link(&(dev->dev.kobj), "master");
7535
7536         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7537                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7538
7539         list_del_rcu(&adj->list);
7540         pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
7541                  adj_dev->name, dev->name, adj_dev->name);
7542         netdev_put(adj_dev, &adj->dev_tracker);
7543         kfree_rcu(adj, rcu);
7544 }
7545
7546 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
7547                                             struct net_device *upper_dev,
7548                                             struct list_head *up_list,
7549                                             struct list_head *down_list,
7550                                             void *private, bool master)
7551 {
7552         int ret;
7553
7554         ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
7555                                            private, master);
7556         if (ret)
7557                 return ret;
7558
7559         ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
7560                                            private, false);
7561         if (ret) {
7562                 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
7563                 return ret;
7564         }
7565
7566         return 0;
7567 }
7568
7569 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
7570                                                struct net_device *upper_dev,
7571                                                u16 ref_nr,
7572                                                struct list_head *up_list,
7573                                                struct list_head *down_list)
7574 {
7575         __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
7576         __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
7577 }
7578
7579 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
7580                                                 struct net_device *upper_dev,
7581                                                 void *private, bool master)
7582 {
7583         return __netdev_adjacent_dev_link_lists(dev, upper_dev,
7584                                                 &dev->adj_list.upper,
7585                                                 &upper_dev->adj_list.lower,
7586                                                 private, master);
7587 }
7588
7589 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
7590                                                    struct net_device *upper_dev)
7591 {
7592         __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
7593                                            &dev->adj_list.upper,
7594                                            &upper_dev->adj_list.lower);
7595 }
7596
7597 static int __netdev_upper_dev_link(struct net_device *dev,
7598                                    struct net_device *upper_dev, bool master,
7599                                    void *upper_priv, void *upper_info,
7600                                    struct netdev_nested_priv *priv,
7601                                    struct netlink_ext_ack *extack)
7602 {
7603         struct netdev_notifier_changeupper_info changeupper_info = {
7604                 .info = {
7605                         .dev = dev,
7606                         .extack = extack,
7607                 },
7608                 .upper_dev = upper_dev,
7609                 .master = master,
7610                 .linking = true,
7611                 .upper_info = upper_info,
7612         };
7613         struct net_device *master_dev;
7614         int ret = 0;
7615
7616         ASSERT_RTNL();
7617
7618         if (dev == upper_dev)
7619                 return -EBUSY;
7620
7621         /* To prevent loops, check if dev is not upper device to upper_dev. */
7622         if (__netdev_has_upper_dev(upper_dev, dev))
7623                 return -EBUSY;
7624
7625         if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
7626                 return -EMLINK;
7627
7628         if (!master) {
7629                 if (__netdev_has_upper_dev(dev, upper_dev))
7630                         return -EEXIST;
7631         } else {
7632                 master_dev = __netdev_master_upper_dev_get(dev);
7633                 if (master_dev)
7634                         return master_dev == upper_dev ? -EEXIST : -EBUSY;
7635         }
7636
7637         ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
7638                                             &changeupper_info.info);
7639         ret = notifier_to_errno(ret);
7640         if (ret)
7641                 return ret;
7642
7643         ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
7644                                                    master);
7645         if (ret)
7646                 return ret;
7647
7648         ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7649                                             &changeupper_info.info);
7650         ret = notifier_to_errno(ret);
7651         if (ret)
7652                 goto rollback;
7653
7654         __netdev_update_upper_level(dev, NULL);
7655         __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7656
7657         __netdev_update_lower_level(upper_dev, priv);
7658         __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
7659                                     priv);
7660
7661         return 0;
7662
7663 rollback:
7664         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7665
7666         return ret;
7667 }
7668
7669 /**
7670  * netdev_upper_dev_link - Add a link to the upper device
7671  * @dev: device
7672  * @upper_dev: new upper device
7673  * @extack: netlink extended ack
7674  *
7675  * Adds a link to device which is upper to this one. The caller must hold
7676  * the RTNL lock. On a failure a negative errno code is returned.
7677  * On success the reference counts are adjusted and the function
7678  * returns zero.
7679  */
7680 int netdev_upper_dev_link(struct net_device *dev,
7681                           struct net_device *upper_dev,
7682                           struct netlink_ext_ack *extack)
7683 {
7684         struct netdev_nested_priv priv = {
7685                 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7686                 .data = NULL,
7687         };
7688
7689         return __netdev_upper_dev_link(dev, upper_dev, false,
7690                                        NULL, NULL, &priv, extack);
7691 }
7692 EXPORT_SYMBOL(netdev_upper_dev_link);
7693
7694 /**
7695  * netdev_master_upper_dev_link - Add a master link to the upper device
7696  * @dev: device
7697  * @upper_dev: new upper device
7698  * @upper_priv: upper device private
7699  * @upper_info: upper info to be passed down via notifier
7700  * @extack: netlink extended ack
7701  *
7702  * Adds a link to device which is upper to this one. In this case, only
7703  * one master upper device can be linked, although other non-master devices
7704  * might be linked as well. The caller must hold the RTNL lock.
7705  * On a failure a negative errno code is returned. On success the reference
7706  * counts are adjusted and the function returns zero.
7707  */
7708 int netdev_master_upper_dev_link(struct net_device *dev,
7709                                  struct net_device *upper_dev,
7710                                  void *upper_priv, void *upper_info,
7711                                  struct netlink_ext_ack *extack)
7712 {
7713         struct netdev_nested_priv priv = {
7714                 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7715                 .data = NULL,
7716         };
7717
7718         return __netdev_upper_dev_link(dev, upper_dev, true,
7719                                        upper_priv, upper_info, &priv, extack);
7720 }
7721 EXPORT_SYMBOL(netdev_master_upper_dev_link);
7722
7723 static void __netdev_upper_dev_unlink(struct net_device *dev,
7724                                       struct net_device *upper_dev,
7725                                       struct netdev_nested_priv *priv)
7726 {
7727         struct netdev_notifier_changeupper_info changeupper_info = {
7728                 .info = {
7729                         .dev = dev,
7730                 },
7731                 .upper_dev = upper_dev,
7732                 .linking = false,
7733         };
7734
7735         ASSERT_RTNL();
7736
7737         changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
7738
7739         call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
7740                                       &changeupper_info.info);
7741
7742         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7743
7744         call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7745                                       &changeupper_info.info);
7746
7747         __netdev_update_upper_level(dev, NULL);
7748         __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7749
7750         __netdev_update_lower_level(upper_dev, priv);
7751         __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
7752                                     priv);
7753 }
7754
7755 /**
7756  * netdev_upper_dev_unlink - Removes a link to upper device
7757  * @dev: device
7758  * @upper_dev: new upper device
7759  *
7760  * Removes a link to device which is upper to this one. The caller must hold
7761  * the RTNL lock.
7762  */
7763 void netdev_upper_dev_unlink(struct net_device *dev,
7764                              struct net_device *upper_dev)
7765 {
7766         struct netdev_nested_priv priv = {
7767                 .flags = NESTED_SYNC_TODO,
7768                 .data = NULL,
7769         };
7770
7771         __netdev_upper_dev_unlink(dev, upper_dev, &priv);
7772 }
7773 EXPORT_SYMBOL(netdev_upper_dev_unlink);
7774
7775 static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
7776                                       struct net_device *lower_dev,
7777                                       bool val)
7778 {
7779         struct netdev_adjacent *adj;
7780
7781         adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
7782         if (adj)
7783                 adj->ignore = val;
7784
7785         adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
7786         if (adj)
7787                 adj->ignore = val;
7788 }
7789
7790 static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
7791                                         struct net_device *lower_dev)
7792 {
7793         __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
7794 }
7795
7796 static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
7797                                        struct net_device *lower_dev)
7798 {
7799         __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
7800 }
7801
7802 int netdev_adjacent_change_prepare(struct net_device *old_dev,
7803                                    struct net_device *new_dev,
7804                                    struct net_device *dev,
7805                                    struct netlink_ext_ack *extack)
7806 {
7807         struct netdev_nested_priv priv = {
7808                 .flags = 0,
7809                 .data = NULL,
7810         };
7811         int err;
7812
7813         if (!new_dev)
7814                 return 0;
7815
7816         if (old_dev && new_dev != old_dev)
7817                 netdev_adjacent_dev_disable(dev, old_dev);
7818         err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
7819                                       extack);
7820         if (err) {
7821                 if (old_dev && new_dev != old_dev)
7822                         netdev_adjacent_dev_enable(dev, old_dev);
7823                 return err;
7824         }
7825
7826         return 0;
7827 }
7828 EXPORT_SYMBOL(netdev_adjacent_change_prepare);
7829
7830 void netdev_adjacent_change_commit(struct net_device *old_dev,
7831                                    struct net_device *new_dev,
7832                                    struct net_device *dev)
7833 {
7834         struct netdev_nested_priv priv = {
7835                 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7836                 .data = NULL,
7837         };
7838
7839         if (!new_dev || !old_dev)
7840                 return;
7841
7842         if (new_dev == old_dev)
7843                 return;
7844
7845         netdev_adjacent_dev_enable(dev, old_dev);
7846         __netdev_upper_dev_unlink(old_dev, dev, &priv);
7847 }
7848 EXPORT_SYMBOL(netdev_adjacent_change_commit);
7849
7850 void netdev_adjacent_change_abort(struct net_device *old_dev,
7851                                   struct net_device *new_dev,
7852                                   struct net_device *dev)
7853 {
7854         struct netdev_nested_priv priv = {
7855                 .flags = 0,
7856                 .data = NULL,
7857         };
7858
7859         if (!new_dev)
7860                 return;
7861
7862         if (old_dev && new_dev != old_dev)
7863                 netdev_adjacent_dev_enable(dev, old_dev);
7864
7865         __netdev_upper_dev_unlink(new_dev, dev, &priv);
7866 }
7867 EXPORT_SYMBOL(netdev_adjacent_change_abort);
7868
7869 /**
7870  * netdev_bonding_info_change - Dispatch event about slave change
7871  * @dev: device
7872  * @bonding_info: info to dispatch
7873  *
7874  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
7875  * The caller must hold the RTNL lock.
7876  */
7877 void netdev_bonding_info_change(struct net_device *dev,
7878                                 struct netdev_bonding_info *bonding_info)
7879 {
7880         struct netdev_notifier_bonding_info info = {
7881                 .info.dev = dev,
7882         };
7883
7884         memcpy(&info.bonding_info, bonding_info,
7885                sizeof(struct netdev_bonding_info));
7886         call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
7887                                       &info.info);
7888 }
7889 EXPORT_SYMBOL(netdev_bonding_info_change);
7890
7891 static int netdev_offload_xstats_enable_l3(struct net_device *dev,
7892                                            struct netlink_ext_ack *extack)
7893 {
7894         struct netdev_notifier_offload_xstats_info info = {
7895                 .info.dev = dev,
7896                 .info.extack = extack,
7897                 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
7898         };
7899         int err;
7900         int rc;
7901
7902         dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3),
7903                                          GFP_KERNEL);
7904         if (!dev->offload_xstats_l3)
7905                 return -ENOMEM;
7906
7907         rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE,
7908                                                   NETDEV_OFFLOAD_XSTATS_DISABLE,
7909                                                   &info.info);
7910         err = notifier_to_errno(rc);
7911         if (err)
7912                 goto free_stats;
7913
7914         return 0;
7915
7916 free_stats:
7917         kfree(dev->offload_xstats_l3);
7918         dev->offload_xstats_l3 = NULL;
7919         return err;
7920 }
7921
7922 int netdev_offload_xstats_enable(struct net_device *dev,
7923                                  enum netdev_offload_xstats_type type,
7924                                  struct netlink_ext_ack *extack)
7925 {
7926         ASSERT_RTNL();
7927
7928         if (netdev_offload_xstats_enabled(dev, type))
7929                 return -EALREADY;
7930
7931         switch (type) {
7932         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
7933                 return netdev_offload_xstats_enable_l3(dev, extack);
7934         }
7935
7936         WARN_ON(1);
7937         return -EINVAL;
7938 }
7939 EXPORT_SYMBOL(netdev_offload_xstats_enable);
7940
7941 static void netdev_offload_xstats_disable_l3(struct net_device *dev)
7942 {
7943         struct netdev_notifier_offload_xstats_info info = {
7944                 .info.dev = dev,
7945                 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
7946         };
7947
7948         call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE,
7949                                       &info.info);
7950         kfree(dev->offload_xstats_l3);
7951         dev->offload_xstats_l3 = NULL;
7952 }
7953
7954 int netdev_offload_xstats_disable(struct net_device *dev,
7955                                   enum netdev_offload_xstats_type type)
7956 {
7957         ASSERT_RTNL();
7958
7959         if (!netdev_offload_xstats_enabled(dev, type))
7960                 return -EALREADY;
7961
7962         switch (type) {
7963         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
7964                 netdev_offload_xstats_disable_l3(dev);
7965                 return 0;
7966         }
7967
7968         WARN_ON(1);
7969         return -EINVAL;
7970 }
7971 EXPORT_SYMBOL(netdev_offload_xstats_disable);
7972
7973 static void netdev_offload_xstats_disable_all(struct net_device *dev)
7974 {
7975         netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
7976 }
7977
7978 static struct rtnl_hw_stats64 *
7979 netdev_offload_xstats_get_ptr(const struct net_device *dev,
7980                               enum netdev_offload_xstats_type type)
7981 {
7982         switch (type) {
7983         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
7984                 return dev->offload_xstats_l3;
7985         }
7986
7987         WARN_ON(1);
7988         return NULL;
7989 }
7990
7991 bool netdev_offload_xstats_enabled(const struct net_device *dev,
7992                                    enum netdev_offload_xstats_type type)
7993 {
7994         ASSERT_RTNL();
7995
7996         return netdev_offload_xstats_get_ptr(dev, type);
7997 }
7998 EXPORT_SYMBOL(netdev_offload_xstats_enabled);
7999
8000 struct netdev_notifier_offload_xstats_ru {
8001         bool used;
8002 };
8003
8004 struct netdev_notifier_offload_xstats_rd {
8005         struct rtnl_hw_stats64 stats;
8006         bool used;
8007 };
8008
8009 static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest,
8010                                   const struct rtnl_hw_stats64 *src)
8011 {
8012         dest->rx_packets          += src->rx_packets;
8013         dest->tx_packets          += src->tx_packets;
8014         dest->rx_bytes            += src->rx_bytes;
8015         dest->tx_bytes            += src->tx_bytes;
8016         dest->rx_errors           += src->rx_errors;
8017         dest->tx_errors           += src->tx_errors;
8018         dest->rx_dropped          += src->rx_dropped;
8019         dest->tx_dropped          += src->tx_dropped;
8020         dest->multicast           += src->multicast;
8021 }
8022
8023 static int netdev_offload_xstats_get_used(struct net_device *dev,
8024                                           enum netdev_offload_xstats_type type,
8025                                           bool *p_used,
8026                                           struct netlink_ext_ack *extack)
8027 {
8028         struct netdev_notifier_offload_xstats_ru report_used = {};
8029         struct netdev_notifier_offload_xstats_info info = {
8030                 .info.dev = dev,
8031                 .info.extack = extack,
8032                 .type = type,
8033                 .report_used = &report_used,
8034         };
8035         int rc;
8036
8037         WARN_ON(!netdev_offload_xstats_enabled(dev, type));
8038         rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED,
8039                                            &info.info);
8040         *p_used = report_used.used;
8041         return notifier_to_errno(rc);
8042 }
8043
8044 static int netdev_offload_xstats_get_stats(struct net_device *dev,
8045                                            enum netdev_offload_xstats_type type,
8046                                            struct rtnl_hw_stats64 *p_stats,
8047                                            bool *p_used,
8048                                            struct netlink_ext_ack *extack)
8049 {
8050         struct netdev_notifier_offload_xstats_rd report_delta = {};
8051         struct netdev_notifier_offload_xstats_info info = {
8052                 .info.dev = dev,
8053                 .info.extack = extack,
8054                 .type = type,
8055                 .report_delta = &report_delta,
8056         };
8057         struct rtnl_hw_stats64 *stats;
8058         int rc;
8059
8060         stats = netdev_offload_xstats_get_ptr(dev, type);
8061         if (WARN_ON(!stats))
8062                 return -EINVAL;
8063
8064         rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
8065                                            &info.info);
8066
8067         /* Cache whatever we got, even if there was an error, otherwise the
8068          * successful stats retrievals would get lost.
8069          */
8070         netdev_hw_stats64_add(stats, &report_delta.stats);
8071
8072         if (p_stats)
8073                 *p_stats = *stats;
8074         *p_used = report_delta.used;
8075
8076         return notifier_to_errno(rc);
8077 }
8078
8079 int netdev_offload_xstats_get(struct net_device *dev,
8080                               enum netdev_offload_xstats_type type,
8081                               struct rtnl_hw_stats64 *p_stats, bool *p_used,
8082                               struct netlink_ext_ack *extack)
8083 {
8084         ASSERT_RTNL();
8085
8086         if (p_stats)
8087                 return netdev_offload_xstats_get_stats(dev, type, p_stats,
8088                                                        p_used, extack);
8089         else
8090                 return netdev_offload_xstats_get_used(dev, type, p_used,
8091                                                       extack);
8092 }
8093 EXPORT_SYMBOL(netdev_offload_xstats_get);
8094
8095 void
8096 netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta,
8097                                    const struct rtnl_hw_stats64 *stats)
8098 {
8099         report_delta->used = true;
8100         netdev_hw_stats64_add(&report_delta->stats, stats);
8101 }
8102 EXPORT_SYMBOL(netdev_offload_xstats_report_delta);
8103
8104 void
8105 netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used)
8106 {
8107         report_used->used = true;
8108 }
8109 EXPORT_SYMBOL(netdev_offload_xstats_report_used);
8110
8111 void netdev_offload_xstats_push_delta(struct net_device *dev,
8112                                       enum netdev_offload_xstats_type type,
8113                                       const struct rtnl_hw_stats64 *p_stats)
8114 {
8115         struct rtnl_hw_stats64 *stats;
8116
8117         ASSERT_RTNL();
8118
8119         stats = netdev_offload_xstats_get_ptr(dev, type);
8120         if (WARN_ON(!stats))
8121                 return;
8122
8123         netdev_hw_stats64_add(stats, p_stats);
8124 }
8125 EXPORT_SYMBOL(netdev_offload_xstats_push_delta);
8126
8127 /**
8128  * netdev_get_xmit_slave - Get the xmit slave of master device
8129  * @dev: device
8130  * @skb: The packet
8131  * @all_slaves: assume all the slaves are active
8132  *
8133  * The reference counters are not incremented so the caller must be
8134  * careful with locks. The caller must hold RCU lock.
8135  * %NULL is returned if no slave is found.
8136  */
8137
8138 struct net_device *netdev_get_xmit_slave(struct net_device *dev,
8139                                          struct sk_buff *skb,
8140                                          bool all_slaves)
8141 {
8142         const struct net_device_ops *ops = dev->netdev_ops;
8143
8144         if (!ops->ndo_get_xmit_slave)
8145                 return NULL;
8146         return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
8147 }
8148 EXPORT_SYMBOL(netdev_get_xmit_slave);
8149
8150 static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
8151                                                   struct sock *sk)
8152 {
8153         const struct net_device_ops *ops = dev->netdev_ops;
8154
8155         if (!ops->ndo_sk_get_lower_dev)
8156                 return NULL;
8157         return ops->ndo_sk_get_lower_dev(dev, sk);
8158 }
8159
8160 /**
8161  * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8162  * @dev: device
8163  * @sk: the socket
8164  *
8165  * %NULL is returned if no lower device is found.
8166  */
8167
8168 struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
8169                                             struct sock *sk)
8170 {
8171         struct net_device *lower;
8172
8173         lower = netdev_sk_get_lower_dev(dev, sk);
8174         while (lower) {
8175                 dev = lower;
8176                 lower = netdev_sk_get_lower_dev(dev, sk);
8177         }
8178
8179         return dev;
8180 }
8181 EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
8182
8183 static void netdev_adjacent_add_links(struct net_device *dev)
8184 {
8185         struct netdev_adjacent *iter;
8186
8187         struct net *net = dev_net(dev);
8188
8189         list_for_each_entry(iter, &dev->adj_list.upper, list) {
8190                 if (!net_eq(net, dev_net(iter->dev)))
8191                         continue;
8192                 netdev_adjacent_sysfs_add(iter->dev, dev,
8193                                           &iter->dev->adj_list.lower);
8194                 netdev_adjacent_sysfs_add(dev, iter->dev,
8195                                           &dev->adj_list.upper);
8196         }
8197
8198         list_for_each_entry(iter, &dev->adj_list.lower, list) {
8199                 if (!net_eq(net, dev_net(iter->dev)))
8200                         continue;
8201                 netdev_adjacent_sysfs_add(iter->dev, dev,
8202                                           &iter->dev->adj_list.upper);
8203                 netdev_adjacent_sysfs_add(dev, iter->dev,
8204                                           &dev->adj_list.lower);
8205         }
8206 }
8207
8208 static void netdev_adjacent_del_links(struct net_device *dev)
8209 {
8210         struct netdev_adjacent *iter;
8211
8212         struct net *net = dev_net(dev);
8213
8214         list_for_each_entry(iter, &dev->adj_list.upper, list) {
8215                 if (!net_eq(net, dev_net(iter->dev)))
8216                         continue;
8217                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8218                                           &iter->dev->adj_list.lower);
8219                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8220                                           &dev->adj_list.upper);
8221         }
8222
8223         list_for_each_entry(iter, &dev->adj_list.lower, list) {
8224                 if (!net_eq(net, dev_net(iter->dev)))
8225                         continue;
8226                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8227                                           &iter->dev->adj_list.upper);
8228                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8229                                           &dev->adj_list.lower);
8230         }
8231 }
8232
8233 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
8234 {
8235         struct netdev_adjacent *iter;
8236
8237         struct net *net = dev_net(dev);
8238
8239         list_for_each_entry(iter, &dev->adj_list.upper, list) {
8240                 if (!net_eq(net, dev_net(iter->dev)))
8241                         continue;
8242                 netdev_adjacent_sysfs_del(iter->dev, oldname,
8243                                           &iter->dev->adj_list.lower);
8244                 netdev_adjacent_sysfs_add(iter->dev, dev,
8245                                           &iter->dev->adj_list.lower);
8246         }
8247
8248         list_for_each_entry(iter, &dev->adj_list.lower, list) {
8249                 if (!net_eq(net, dev_net(iter->dev)))
8250                         continue;
8251                 netdev_adjacent_sysfs_del(iter->dev, oldname,
8252                                           &iter->dev->adj_list.upper);
8253                 netdev_adjacent_sysfs_add(iter->dev, dev,
8254                                           &iter->dev->adj_list.upper);
8255         }
8256 }
8257
8258 void *netdev_lower_dev_get_private(struct net_device *dev,
8259                                    struct net_device *lower_dev)
8260 {
8261         struct netdev_adjacent *lower;
8262
8263         if (!lower_dev)
8264                 return NULL;
8265         lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
8266         if (!lower)
8267                 return NULL;
8268
8269         return lower->private;
8270 }
8271 EXPORT_SYMBOL(netdev_lower_dev_get_private);
8272
8273
8274 /**
8275  * netdev_lower_state_changed - Dispatch event about lower device state change
8276  * @lower_dev: device
8277  * @lower_state_info: state to dispatch
8278  *
8279  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
8280  * The caller must hold the RTNL lock.
8281  */
8282 void netdev_lower_state_changed(struct net_device *lower_dev,
8283                                 void *lower_state_info)
8284 {
8285         struct netdev_notifier_changelowerstate_info changelowerstate_info = {
8286                 .info.dev = lower_dev,
8287         };
8288
8289         ASSERT_RTNL();
8290         changelowerstate_info.lower_state_info = lower_state_info;
8291         call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
8292                                       &changelowerstate_info.info);
8293 }
8294 EXPORT_SYMBOL(netdev_lower_state_changed);
8295
8296 static void dev_change_rx_flags(struct net_device *dev, int flags)
8297 {
8298         const struct net_device_ops *ops = dev->netdev_ops;
8299
8300         if (ops->ndo_change_rx_flags)
8301                 ops->ndo_change_rx_flags(dev, flags);
8302 }
8303
8304 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
8305 {
8306         unsigned int old_flags = dev->flags;
8307         kuid_t uid;
8308         kgid_t gid;
8309
8310         ASSERT_RTNL();
8311
8312         dev->flags |= IFF_PROMISC;
8313         dev->promiscuity += inc;
8314         if (dev->promiscuity == 0) {
8315                 /*
8316                  * Avoid overflow.
8317                  * If inc causes overflow, untouch promisc and return error.
8318                  */
8319                 if (inc < 0)
8320                         dev->flags &= ~IFF_PROMISC;
8321                 else {
8322                         dev->promiscuity -= inc;
8323                         netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n");
8324                         return -EOVERFLOW;
8325                 }
8326         }
8327         if (dev->flags != old_flags) {
8328                 netdev_info(dev, "%s promiscuous mode\n",
8329                             dev->flags & IFF_PROMISC ? "entered" : "left");
8330                 if (audit_enabled) {
8331                         current_uid_gid(&uid, &gid);
8332                         audit_log(audit_context(), GFP_ATOMIC,
8333                                   AUDIT_ANOM_PROMISCUOUS,
8334                                   "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
8335                                   dev->name, (dev->flags & IFF_PROMISC),
8336                                   (old_flags & IFF_PROMISC),
8337                                   from_kuid(&init_user_ns, audit_get_loginuid(current)),
8338                                   from_kuid(&init_user_ns, uid),
8339                                   from_kgid(&init_user_ns, gid),
8340                                   audit_get_sessionid(current));
8341                 }
8342
8343                 dev_change_rx_flags(dev, IFF_PROMISC);
8344         }
8345         if (notify)
8346                 __dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL);
8347         return 0;
8348 }
8349
8350 /**
8351  *      dev_set_promiscuity     - update promiscuity count on a device
8352  *      @dev: device
8353  *      @inc: modifier
8354  *
8355  *      Add or remove promiscuity from a device. While the count in the device
8356  *      remains above zero the interface remains promiscuous. Once it hits zero
8357  *      the device reverts back to normal filtering operation. A negative inc
8358  *      value is used to drop promiscuity on the device.
8359  *      Return 0 if successful or a negative errno code on error.
8360  */
8361 int dev_set_promiscuity(struct net_device *dev, int inc)
8362 {
8363         unsigned int old_flags = dev->flags;
8364         int err;
8365
8366         err = __dev_set_promiscuity(dev, inc, true);
8367         if (err < 0)
8368                 return err;
8369         if (dev->flags != old_flags)
8370                 dev_set_rx_mode(dev);
8371         return err;
8372 }
8373 EXPORT_SYMBOL(dev_set_promiscuity);
8374
8375 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
8376 {
8377         unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
8378
8379         ASSERT_RTNL();
8380
8381         dev->flags |= IFF_ALLMULTI;
8382         dev->allmulti += inc;
8383         if (dev->allmulti == 0) {
8384                 /*
8385                  * Avoid overflow.
8386                  * If inc causes overflow, untouch allmulti and return error.
8387                  */
8388                 if (inc < 0)
8389                         dev->flags &= ~IFF_ALLMULTI;
8390                 else {
8391                         dev->allmulti -= inc;
8392                         netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n");
8393                         return -EOVERFLOW;
8394                 }
8395         }
8396         if (dev->flags ^ old_flags) {
8397                 netdev_info(dev, "%s allmulticast mode\n",
8398                             dev->flags & IFF_ALLMULTI ? "entered" : "left");
8399                 dev_change_rx_flags(dev, IFF_ALLMULTI);
8400                 dev_set_rx_mode(dev);
8401                 if (notify)
8402                         __dev_notify_flags(dev, old_flags,
8403                                            dev->gflags ^ old_gflags, 0, NULL);
8404         }
8405         return 0;
8406 }
8407
8408 /**
8409  *      dev_set_allmulti        - update allmulti count on a device
8410  *      @dev: device
8411  *      @inc: modifier
8412  *
8413  *      Add or remove reception of all multicast frames to a device. While the
8414  *      count in the device remains above zero the interface remains listening
8415  *      to all interfaces. Once it hits zero the device reverts back to normal
8416  *      filtering operation. A negative @inc value is used to drop the counter
8417  *      when releasing a resource needing all multicasts.
8418  *      Return 0 if successful or a negative errno code on error.
8419  */
8420
8421 int dev_set_allmulti(struct net_device *dev, int inc)
8422 {
8423         return __dev_set_allmulti(dev, inc, true);
8424 }
8425 EXPORT_SYMBOL(dev_set_allmulti);
8426
8427 /*
8428  *      Upload unicast and multicast address lists to device and
8429  *      configure RX filtering. When the device doesn't support unicast
8430  *      filtering it is put in promiscuous mode while unicast addresses
8431  *      are present.
8432  */
8433 void __dev_set_rx_mode(struct net_device *dev)
8434 {
8435         const struct net_device_ops *ops = dev->netdev_ops;
8436
8437         /* dev_open will call this function so the list will stay sane. */
8438         if (!(dev->flags&IFF_UP))
8439                 return;
8440
8441         if (!netif_device_present(dev))
8442                 return;
8443
8444         if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
8445                 /* Unicast addresses changes may only happen under the rtnl,
8446                  * therefore calling __dev_set_promiscuity here is safe.
8447                  */
8448                 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
8449                         __dev_set_promiscuity(dev, 1, false);
8450                         dev->uc_promisc = true;
8451                 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
8452                         __dev_set_promiscuity(dev, -1, false);
8453                         dev->uc_promisc = false;
8454                 }
8455         }
8456
8457         if (ops->ndo_set_rx_mode)
8458                 ops->ndo_set_rx_mode(dev);
8459 }
8460
8461 void dev_set_rx_mode(struct net_device *dev)
8462 {
8463         netif_addr_lock_bh(dev);
8464         __dev_set_rx_mode(dev);
8465         netif_addr_unlock_bh(dev);
8466 }
8467
8468 /**
8469  *      dev_get_flags - get flags reported to userspace
8470  *      @dev: device
8471  *
8472  *      Get the combination of flag bits exported through APIs to userspace.
8473  */
8474 unsigned int dev_get_flags(const struct net_device *dev)
8475 {
8476         unsigned int flags;
8477
8478         flags = (dev->flags & ~(IFF_PROMISC |
8479                                 IFF_ALLMULTI |
8480                                 IFF_RUNNING |
8481                                 IFF_LOWER_UP |
8482                                 IFF_DORMANT)) |
8483                 (dev->gflags & (IFF_PROMISC |
8484                                 IFF_ALLMULTI));
8485
8486         if (netif_running(dev)) {
8487                 if (netif_oper_up(dev))
8488                         flags |= IFF_RUNNING;
8489                 if (netif_carrier_ok(dev))
8490                         flags |= IFF_LOWER_UP;
8491                 if (netif_dormant(dev))
8492                         flags |= IFF_DORMANT;
8493         }
8494
8495         return flags;
8496 }
8497 EXPORT_SYMBOL(dev_get_flags);
8498
8499 int __dev_change_flags(struct net_device *dev, unsigned int flags,
8500                        struct netlink_ext_ack *extack)
8501 {
8502         unsigned int old_flags = dev->flags;
8503         int ret;
8504
8505         ASSERT_RTNL();
8506
8507         /*
8508          *      Set the flags on our device.
8509          */
8510
8511         dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
8512                                IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
8513                                IFF_AUTOMEDIA)) |
8514                      (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
8515                                     IFF_ALLMULTI));
8516
8517         /*
8518          *      Load in the correct multicast list now the flags have changed.
8519          */
8520
8521         if ((old_flags ^ flags) & IFF_MULTICAST)
8522                 dev_change_rx_flags(dev, IFF_MULTICAST);
8523
8524         dev_set_rx_mode(dev);
8525
8526         /*
8527          *      Have we downed the interface. We handle IFF_UP ourselves
8528          *      according to user attempts to set it, rather than blindly
8529          *      setting it.
8530          */
8531
8532         ret = 0;
8533         if ((old_flags ^ flags) & IFF_UP) {
8534                 if (old_flags & IFF_UP)
8535                         __dev_close(dev);
8536                 else
8537                         ret = __dev_open(dev, extack);
8538         }
8539
8540         if ((flags ^ dev->gflags) & IFF_PROMISC) {
8541                 int inc = (flags & IFF_PROMISC) ? 1 : -1;
8542                 unsigned int old_flags = dev->flags;
8543
8544                 dev->gflags ^= IFF_PROMISC;
8545
8546                 if (__dev_set_promiscuity(dev, inc, false) >= 0)
8547                         if (dev->flags != old_flags)
8548                                 dev_set_rx_mode(dev);
8549         }
8550
8551         /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
8552          * is important. Some (broken) drivers set IFF_PROMISC, when
8553          * IFF_ALLMULTI is requested not asking us and not reporting.
8554          */
8555         if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
8556                 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
8557
8558                 dev->gflags ^= IFF_ALLMULTI;
8559                 __dev_set_allmulti(dev, inc, false);
8560         }
8561
8562         return ret;
8563 }
8564
8565 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
8566                         unsigned int gchanges, u32 portid,
8567                         const struct nlmsghdr *nlh)
8568 {
8569         unsigned int changes = dev->flags ^ old_flags;
8570
8571         if (gchanges)
8572                 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh);
8573
8574         if (changes & IFF_UP) {
8575                 if (dev->flags & IFF_UP)
8576                         call_netdevice_notifiers(NETDEV_UP, dev);
8577                 else
8578                         call_netdevice_notifiers(NETDEV_DOWN, dev);
8579         }
8580
8581         if (dev->flags & IFF_UP &&
8582             (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
8583                 struct netdev_notifier_change_info change_info = {
8584                         .info = {
8585                                 .dev = dev,
8586                         },
8587                         .flags_changed = changes,
8588                 };
8589
8590                 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
8591         }
8592 }
8593
8594 /**
8595  *      dev_change_flags - change device settings
8596  *      @dev: device
8597  *      @flags: device state flags
8598  *      @extack: netlink extended ack
8599  *
8600  *      Change settings on device based state flags. The flags are
8601  *      in the userspace exported format.
8602  */
8603 int dev_change_flags(struct net_device *dev, unsigned int flags,
8604                      struct netlink_ext_ack *extack)
8605 {
8606         int ret;
8607         unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
8608
8609         ret = __dev_change_flags(dev, flags, extack);
8610         if (ret < 0)
8611                 return ret;
8612
8613         changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
8614         __dev_notify_flags(dev, old_flags, changes, 0, NULL);
8615         return ret;
8616 }
8617 EXPORT_SYMBOL(dev_change_flags);
8618
8619 int __dev_set_mtu(struct net_device *dev, int new_mtu)
8620 {
8621         const struct net_device_ops *ops = dev->netdev_ops;
8622
8623         if (ops->ndo_change_mtu)
8624                 return ops->ndo_change_mtu(dev, new_mtu);
8625
8626         /* Pairs with all the lockless reads of dev->mtu in the stack */
8627         WRITE_ONCE(dev->mtu, new_mtu);
8628         return 0;
8629 }
8630 EXPORT_SYMBOL(__dev_set_mtu);
8631
8632 int dev_validate_mtu(struct net_device *dev, int new_mtu,
8633                      struct netlink_ext_ack *extack)
8634 {
8635         /* MTU must be positive, and in range */
8636         if (new_mtu < 0 || new_mtu < dev->min_mtu) {
8637                 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
8638                 return -EINVAL;
8639         }
8640
8641         if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
8642                 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
8643                 return -EINVAL;
8644         }
8645         return 0;
8646 }
8647
8648 /**
8649  *      dev_set_mtu_ext - Change maximum transfer unit
8650  *      @dev: device
8651  *      @new_mtu: new transfer unit
8652  *      @extack: netlink extended ack
8653  *
8654  *      Change the maximum transfer size of the network device.
8655  */
8656 int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
8657                     struct netlink_ext_ack *extack)
8658 {
8659         int err, orig_mtu;
8660
8661         if (new_mtu == dev->mtu)
8662                 return 0;
8663
8664         err = dev_validate_mtu(dev, new_mtu, extack);
8665         if (err)
8666                 return err;
8667
8668         if (!netif_device_present(dev))
8669                 return -ENODEV;
8670
8671         err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
8672         err = notifier_to_errno(err);
8673         if (err)
8674                 return err;
8675
8676         orig_mtu = dev->mtu;
8677         err = __dev_set_mtu(dev, new_mtu);
8678
8679         if (!err) {
8680                 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8681                                                    orig_mtu);
8682                 err = notifier_to_errno(err);
8683                 if (err) {
8684                         /* setting mtu back and notifying everyone again,
8685                          * so that they have a chance to revert changes.
8686                          */
8687                         __dev_set_mtu(dev, orig_mtu);
8688                         call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8689                                                      new_mtu);
8690                 }
8691         }
8692         return err;
8693 }
8694
8695 int dev_set_mtu(struct net_device *dev, int new_mtu)
8696 {
8697         struct netlink_ext_ack extack;
8698         int err;
8699
8700         memset(&extack, 0, sizeof(extack));
8701         err = dev_set_mtu_ext(dev, new_mtu, &extack);
8702         if (err && extack._msg)
8703                 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
8704         return err;
8705 }
8706 EXPORT_SYMBOL(dev_set_mtu);
8707
8708 /**
8709  *      dev_change_tx_queue_len - Change TX queue length of a netdevice
8710  *      @dev: device
8711  *      @new_len: new tx queue length
8712  */
8713 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
8714 {
8715         unsigned int orig_len = dev->tx_queue_len;
8716         int res;
8717
8718         if (new_len != (unsigned int)new_len)
8719                 return -ERANGE;
8720
8721         if (new_len != orig_len) {
8722                 dev->tx_queue_len = new_len;
8723                 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
8724                 res = notifier_to_errno(res);
8725                 if (res)
8726                         goto err_rollback;
8727                 res = dev_qdisc_change_tx_queue_len(dev);
8728                 if (res)
8729                         goto err_rollback;
8730         }
8731
8732         return 0;
8733
8734 err_rollback:
8735         netdev_err(dev, "refused to change device tx_queue_len\n");
8736         dev->tx_queue_len = orig_len;
8737         return res;
8738 }
8739
8740 /**
8741  *      dev_set_group - Change group this device belongs to
8742  *      @dev: device
8743  *      @new_group: group this device should belong to
8744  */
8745 void dev_set_group(struct net_device *dev, int new_group)
8746 {
8747         dev->group = new_group;
8748 }
8749
8750 /**
8751  *      dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
8752  *      @dev: device
8753  *      @addr: new address
8754  *      @extack: netlink extended ack
8755  */
8756 int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
8757                               struct netlink_ext_ack *extack)
8758 {
8759         struct netdev_notifier_pre_changeaddr_info info = {
8760                 .info.dev = dev,
8761                 .info.extack = extack,
8762                 .dev_addr = addr,
8763         };
8764         int rc;
8765
8766         rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
8767         return notifier_to_errno(rc);
8768 }
8769 EXPORT_SYMBOL(dev_pre_changeaddr_notify);
8770
8771 /**
8772  *      dev_set_mac_address - Change Media Access Control Address
8773  *      @dev: device
8774  *      @sa: new address
8775  *      @extack: netlink extended ack
8776  *
8777  *      Change the hardware (MAC) address of the device
8778  */
8779 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
8780                         struct netlink_ext_ack *extack)
8781 {
8782         const struct net_device_ops *ops = dev->netdev_ops;
8783         int err;
8784
8785         if (!ops->ndo_set_mac_address)
8786                 return -EOPNOTSUPP;
8787         if (sa->sa_family != dev->type)
8788                 return -EINVAL;
8789         if (!netif_device_present(dev))
8790                 return -ENODEV;
8791         err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
8792         if (err)
8793                 return err;
8794         err = ops->ndo_set_mac_address(dev, sa);
8795         if (err)
8796                 return err;
8797         dev->addr_assign_type = NET_ADDR_SET;
8798         call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
8799         add_device_randomness(dev->dev_addr, dev->addr_len);
8800         return 0;
8801 }
8802 EXPORT_SYMBOL(dev_set_mac_address);
8803
8804 static DECLARE_RWSEM(dev_addr_sem);
8805
8806 int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
8807                              struct netlink_ext_ack *extack)
8808 {
8809         int ret;
8810
8811         down_write(&dev_addr_sem);
8812         ret = dev_set_mac_address(dev, sa, extack);
8813         up_write(&dev_addr_sem);
8814         return ret;
8815 }
8816 EXPORT_SYMBOL(dev_set_mac_address_user);
8817
8818 int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
8819 {
8820         size_t size = sizeof(sa->sa_data_min);
8821         struct net_device *dev;
8822         int ret = 0;
8823
8824         down_read(&dev_addr_sem);
8825         rcu_read_lock();
8826
8827         dev = dev_get_by_name_rcu(net, dev_name);
8828         if (!dev) {
8829                 ret = -ENODEV;
8830                 goto unlock;
8831         }
8832         if (!dev->addr_len)
8833                 memset(sa->sa_data, 0, size);
8834         else
8835                 memcpy(sa->sa_data, dev->dev_addr,
8836                        min_t(size_t, size, dev->addr_len));
8837         sa->sa_family = dev->type;
8838
8839 unlock:
8840         rcu_read_unlock();
8841         up_read(&dev_addr_sem);
8842         return ret;
8843 }
8844 EXPORT_SYMBOL(dev_get_mac_address);
8845
8846 /**
8847  *      dev_change_carrier - Change device carrier
8848  *      @dev: device
8849  *      @new_carrier: new value
8850  *
8851  *      Change device carrier
8852  */
8853 int dev_change_carrier(struct net_device *dev, bool new_carrier)
8854 {
8855         const struct net_device_ops *ops = dev->netdev_ops;
8856
8857         if (!ops->ndo_change_carrier)
8858                 return -EOPNOTSUPP;
8859         if (!netif_device_present(dev))
8860                 return -ENODEV;
8861         return ops->ndo_change_carrier(dev, new_carrier);
8862 }
8863
8864 /**
8865  *      dev_get_phys_port_id - Get device physical port ID
8866  *      @dev: device
8867  *      @ppid: port ID
8868  *
8869  *      Get device physical port ID
8870  */
8871 int dev_get_phys_port_id(struct net_device *dev,
8872                          struct netdev_phys_item_id *ppid)
8873 {
8874         const struct net_device_ops *ops = dev->netdev_ops;
8875
8876         if (!ops->ndo_get_phys_port_id)
8877                 return -EOPNOTSUPP;
8878         return ops->ndo_get_phys_port_id(dev, ppid);
8879 }
8880
8881 /**
8882  *      dev_get_phys_port_name - Get device physical port name
8883  *      @dev: device
8884  *      @name: port name
8885  *      @len: limit of bytes to copy to name
8886  *
8887  *      Get device physical port name
8888  */
8889 int dev_get_phys_port_name(struct net_device *dev,
8890                            char *name, size_t len)
8891 {
8892         const struct net_device_ops *ops = dev->netdev_ops;
8893         int err;
8894
8895         if (ops->ndo_get_phys_port_name) {
8896                 err = ops->ndo_get_phys_port_name(dev, name, len);
8897                 if (err != -EOPNOTSUPP)
8898                         return err;
8899         }
8900         return devlink_compat_phys_port_name_get(dev, name, len);
8901 }
8902
8903 /**
8904  *      dev_get_port_parent_id - Get the device's port parent identifier
8905  *      @dev: network device
8906  *      @ppid: pointer to a storage for the port's parent identifier
8907  *      @recurse: allow/disallow recursion to lower devices
8908  *
8909  *      Get the devices's port parent identifier
8910  */
8911 int dev_get_port_parent_id(struct net_device *dev,
8912                            struct netdev_phys_item_id *ppid,
8913                            bool recurse)
8914 {
8915         const struct net_device_ops *ops = dev->netdev_ops;
8916         struct netdev_phys_item_id first = { };
8917         struct net_device *lower_dev;
8918         struct list_head *iter;
8919         int err;
8920
8921         if (ops->ndo_get_port_parent_id) {
8922                 err = ops->ndo_get_port_parent_id(dev, ppid);
8923                 if (err != -EOPNOTSUPP)
8924                         return err;
8925         }
8926
8927         err = devlink_compat_switch_id_get(dev, ppid);
8928         if (!recurse || err != -EOPNOTSUPP)
8929                 return err;
8930
8931         netdev_for_each_lower_dev(dev, lower_dev, iter) {
8932                 err = dev_get_port_parent_id(lower_dev, ppid, true);
8933                 if (err)
8934                         break;
8935                 if (!first.id_len)
8936                         first = *ppid;
8937                 else if (memcmp(&first, ppid, sizeof(*ppid)))
8938                         return -EOPNOTSUPP;
8939         }
8940
8941         return err;
8942 }
8943 EXPORT_SYMBOL(dev_get_port_parent_id);
8944
8945 /**
8946  *      netdev_port_same_parent_id - Indicate if two network devices have
8947  *      the same port parent identifier
8948  *      @a: first network device
8949  *      @b: second network device
8950  */
8951 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
8952 {
8953         struct netdev_phys_item_id a_id = { };
8954         struct netdev_phys_item_id b_id = { };
8955
8956         if (dev_get_port_parent_id(a, &a_id, true) ||
8957             dev_get_port_parent_id(b, &b_id, true))
8958                 return false;
8959
8960         return netdev_phys_item_id_same(&a_id, &b_id);
8961 }
8962 EXPORT_SYMBOL(netdev_port_same_parent_id);
8963
8964 /**
8965  *      dev_change_proto_down - set carrier according to proto_down.
8966  *
8967  *      @dev: device
8968  *      @proto_down: new value
8969  */
8970 int dev_change_proto_down(struct net_device *dev, bool proto_down)
8971 {
8972         if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN))
8973                 return -EOPNOTSUPP;
8974         if (!netif_device_present(dev))
8975                 return -ENODEV;
8976         if (proto_down)
8977                 netif_carrier_off(dev);
8978         else
8979                 netif_carrier_on(dev);
8980         dev->proto_down = proto_down;
8981         return 0;
8982 }
8983
8984 /**
8985  *      dev_change_proto_down_reason - proto down reason
8986  *
8987  *      @dev: device
8988  *      @mask: proto down mask
8989  *      @value: proto down value
8990  */
8991 void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
8992                                   u32 value)
8993 {
8994         int b;
8995
8996         if (!mask) {
8997                 dev->proto_down_reason = value;
8998         } else {
8999                 for_each_set_bit(b, &mask, 32) {
9000                         if (value & (1 << b))
9001                                 dev->proto_down_reason |= BIT(b);
9002                         else
9003                                 dev->proto_down_reason &= ~BIT(b);
9004                 }
9005         }
9006 }
9007
9008 struct bpf_xdp_link {
9009         struct bpf_link link;
9010         struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
9011         int flags;
9012 };
9013
9014 static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
9015 {
9016         if (flags & XDP_FLAGS_HW_MODE)
9017                 return XDP_MODE_HW;
9018         if (flags & XDP_FLAGS_DRV_MODE)
9019                 return XDP_MODE_DRV;
9020         if (flags & XDP_FLAGS_SKB_MODE)
9021                 return XDP_MODE_SKB;
9022         return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
9023 }
9024
9025 static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
9026 {
9027         switch (mode) {
9028         case XDP_MODE_SKB:
9029                 return generic_xdp_install;
9030         case XDP_MODE_DRV:
9031         case XDP_MODE_HW:
9032                 return dev->netdev_ops->ndo_bpf;
9033         default:
9034                 return NULL;
9035         }
9036 }
9037
9038 static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
9039                                          enum bpf_xdp_mode mode)
9040 {
9041         return dev->xdp_state[mode].link;
9042 }
9043
9044 static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
9045                                      enum bpf_xdp_mode mode)
9046 {
9047         struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
9048
9049         if (link)
9050                 return link->link.prog;
9051         return dev->xdp_state[mode].prog;
9052 }
9053
9054 u8 dev_xdp_prog_count(struct net_device *dev)
9055 {
9056         u8 count = 0;
9057         int i;
9058
9059         for (i = 0; i < __MAX_XDP_MODE; i++)
9060                 if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
9061                         count++;
9062         return count;
9063 }
9064 EXPORT_SYMBOL_GPL(dev_xdp_prog_count);
9065
9066 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
9067 {
9068         struct bpf_prog *prog = dev_xdp_prog(dev, mode);
9069
9070         return prog ? prog->aux->id : 0;
9071 }
9072
9073 static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
9074                              struct bpf_xdp_link *link)
9075 {
9076         dev->xdp_state[mode].link = link;
9077         dev->xdp_state[mode].prog = NULL;
9078 }
9079
9080 static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
9081                              struct bpf_prog *prog)
9082 {
9083         dev->xdp_state[mode].link = NULL;
9084         dev->xdp_state[mode].prog = prog;
9085 }
9086
9087 static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
9088                            bpf_op_t bpf_op, struct netlink_ext_ack *extack,
9089                            u32 flags, struct bpf_prog *prog)
9090 {
9091         struct netdev_bpf xdp;
9092         int err;
9093
9094         memset(&xdp, 0, sizeof(xdp));
9095         xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
9096         xdp.extack = extack;
9097         xdp.flags = flags;
9098         xdp.prog = prog;
9099
9100         /* Drivers assume refcnt is already incremented (i.e, prog pointer is
9101          * "moved" into driver), so they don't increment it on their own, but
9102          * they do decrement refcnt when program is detached or replaced.
9103          * Given net_device also owns link/prog, we need to bump refcnt here
9104          * to prevent drivers from underflowing it.
9105          */
9106         if (prog)
9107                 bpf_prog_inc(prog);
9108         err = bpf_op(dev, &xdp);
9109         if (err) {
9110                 if (prog)
9111                         bpf_prog_put(prog);
9112                 return err;
9113         }
9114
9115         if (mode != XDP_MODE_HW)
9116                 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
9117
9118         return 0;
9119 }
9120
9121 static void dev_xdp_uninstall(struct net_device *dev)
9122 {
9123         struct bpf_xdp_link *link;
9124         struct bpf_prog *prog;
9125         enum bpf_xdp_mode mode;
9126         bpf_op_t bpf_op;
9127
9128         ASSERT_RTNL();
9129
9130         for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
9131                 prog = dev_xdp_prog(dev, mode);
9132                 if (!prog)
9133                         continue;
9134
9135                 bpf_op = dev_xdp_bpf_op(dev, mode);
9136                 if (!bpf_op)
9137                         continue;
9138
9139                 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9140
9141                 /* auto-detach link from net device */
9142                 link = dev_xdp_link(dev, mode);
9143                 if (link)
9144                         link->dev = NULL;
9145                 else
9146                         bpf_prog_put(prog);
9147
9148                 dev_xdp_set_link(dev, mode, NULL);
9149         }
9150 }
9151
9152 static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
9153                           struct bpf_xdp_link *link, struct bpf_prog *new_prog,
9154                           struct bpf_prog *old_prog, u32 flags)
9155 {
9156         unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
9157         struct bpf_prog *cur_prog;
9158         struct net_device *upper;
9159         struct list_head *iter;
9160         enum bpf_xdp_mode mode;
9161         bpf_op_t bpf_op;
9162         int err;
9163
9164         ASSERT_RTNL();
9165
9166         /* either link or prog attachment, never both */
9167         if (link && (new_prog || old_prog))
9168                 return -EINVAL;
9169         /* link supports only XDP mode flags */
9170         if (link && (flags & ~XDP_FLAGS_MODES)) {
9171                 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
9172                 return -EINVAL;
9173         }
9174         /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
9175         if (num_modes > 1) {
9176                 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
9177                 return -EINVAL;
9178         }
9179         /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
9180         if (!num_modes && dev_xdp_prog_count(dev) > 1) {
9181                 NL_SET_ERR_MSG(extack,
9182                                "More than one program loaded, unset mode is ambiguous");
9183                 return -EINVAL;
9184         }
9185         /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
9186         if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
9187                 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
9188                 return -EINVAL;
9189         }
9190
9191         mode = dev_xdp_mode(dev, flags);
9192         /* can't replace attached link */
9193         if (dev_xdp_link(dev, mode)) {
9194                 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
9195                 return -EBUSY;
9196         }
9197
9198         /* don't allow if an upper device already has a program */
9199         netdev_for_each_upper_dev_rcu(dev, upper, iter) {
9200                 if (dev_xdp_prog_count(upper) > 0) {
9201                         NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program");
9202                         return -EEXIST;
9203                 }
9204         }
9205
9206         cur_prog = dev_xdp_prog(dev, mode);
9207         /* can't replace attached prog with link */
9208         if (link && cur_prog) {
9209                 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
9210                 return -EBUSY;
9211         }
9212         if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
9213                 NL_SET_ERR_MSG(extack, "Active program does not match expected");
9214                 return -EEXIST;
9215         }
9216
9217         /* put effective new program into new_prog */
9218         if (link)
9219                 new_prog = link->link.prog;
9220
9221         if (new_prog) {
9222                 bool offload = mode == XDP_MODE_HW;
9223                 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
9224                                                ? XDP_MODE_DRV : XDP_MODE_SKB;
9225
9226                 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
9227                         NL_SET_ERR_MSG(extack, "XDP program already attached");
9228                         return -EBUSY;
9229                 }
9230                 if (!offload && dev_xdp_prog(dev, other_mode)) {
9231                         NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
9232                         return -EEXIST;
9233                 }
9234                 if (!offload && bpf_prog_is_offloaded(new_prog->aux)) {
9235                         NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported");
9236                         return -EINVAL;
9237                 }
9238                 if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) {
9239                         NL_SET_ERR_MSG(extack, "Program bound to different device");
9240                         return -EINVAL;
9241                 }
9242                 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
9243                         NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
9244                         return -EINVAL;
9245                 }
9246                 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
9247                         NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
9248                         return -EINVAL;
9249                 }
9250         }
9251
9252         /* don't call drivers if the effective program didn't change */
9253         if (new_prog != cur_prog) {
9254                 bpf_op = dev_xdp_bpf_op(dev, mode);
9255                 if (!bpf_op) {
9256                         NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
9257                         return -EOPNOTSUPP;
9258                 }
9259
9260                 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
9261                 if (err)
9262                         return err;
9263         }
9264
9265         if (link)
9266                 dev_xdp_set_link(dev, mode, link);
9267         else
9268                 dev_xdp_set_prog(dev, mode, new_prog);
9269         if (cur_prog)
9270                 bpf_prog_put(cur_prog);
9271
9272         return 0;
9273 }
9274
9275 static int dev_xdp_attach_link(struct net_device *dev,
9276                                struct netlink_ext_ack *extack,
9277                                struct bpf_xdp_link *link)
9278 {
9279         return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
9280 }
9281
9282 static int dev_xdp_detach_link(struct net_device *dev,
9283                                struct netlink_ext_ack *extack,
9284                                struct bpf_xdp_link *link)
9285 {
9286         enum bpf_xdp_mode mode;
9287         bpf_op_t bpf_op;
9288
9289         ASSERT_RTNL();
9290
9291         mode = dev_xdp_mode(dev, link->flags);
9292         if (dev_xdp_link(dev, mode) != link)
9293                 return -EINVAL;
9294
9295         bpf_op = dev_xdp_bpf_op(dev, mode);
9296         WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9297         dev_xdp_set_link(dev, mode, NULL);
9298         return 0;
9299 }
9300
9301 static void bpf_xdp_link_release(struct bpf_link *link)
9302 {
9303         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9304
9305         rtnl_lock();
9306
9307         /* if racing with net_device's tear down, xdp_link->dev might be
9308          * already NULL, in which case link was already auto-detached
9309          */
9310         if (xdp_link->dev) {
9311                 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
9312                 xdp_link->dev = NULL;
9313         }
9314
9315         rtnl_unlock();
9316 }
9317
9318 static int bpf_xdp_link_detach(struct bpf_link *link)
9319 {
9320         bpf_xdp_link_release(link);
9321         return 0;
9322 }
9323
9324 static void bpf_xdp_link_dealloc(struct bpf_link *link)
9325 {
9326         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9327
9328         kfree(xdp_link);
9329 }
9330
9331 static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
9332                                      struct seq_file *seq)
9333 {
9334         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9335         u32 ifindex = 0;
9336
9337         rtnl_lock();
9338         if (xdp_link->dev)
9339                 ifindex = xdp_link->dev->ifindex;
9340         rtnl_unlock();
9341
9342         seq_printf(seq, "ifindex:\t%u\n", ifindex);
9343 }
9344
9345 static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
9346                                        struct bpf_link_info *info)
9347 {
9348         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9349         u32 ifindex = 0;
9350
9351         rtnl_lock();
9352         if (xdp_link->dev)
9353                 ifindex = xdp_link->dev->ifindex;
9354         rtnl_unlock();
9355
9356         info->xdp.ifindex = ifindex;
9357         return 0;
9358 }
9359
9360 static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
9361                                struct bpf_prog *old_prog)
9362 {
9363         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9364         enum bpf_xdp_mode mode;
9365         bpf_op_t bpf_op;
9366         int err = 0;
9367
9368         rtnl_lock();
9369
9370         /* link might have been auto-released already, so fail */
9371         if (!xdp_link->dev) {
9372                 err = -ENOLINK;
9373                 goto out_unlock;
9374         }
9375
9376         if (old_prog && link->prog != old_prog) {
9377                 err = -EPERM;
9378                 goto out_unlock;
9379         }
9380         old_prog = link->prog;
9381         if (old_prog->type != new_prog->type ||
9382             old_prog->expected_attach_type != new_prog->expected_attach_type) {
9383                 err = -EINVAL;
9384                 goto out_unlock;
9385         }
9386
9387         if (old_prog == new_prog) {
9388                 /* no-op, don't disturb drivers */
9389                 bpf_prog_put(new_prog);
9390                 goto out_unlock;
9391         }
9392
9393         mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
9394         bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
9395         err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
9396                               xdp_link->flags, new_prog);
9397         if (err)
9398                 goto out_unlock;
9399
9400         old_prog = xchg(&link->prog, new_prog);
9401         bpf_prog_put(old_prog);
9402
9403 out_unlock:
9404         rtnl_unlock();
9405         return err;
9406 }
9407
9408 static const struct bpf_link_ops bpf_xdp_link_lops = {
9409         .release = bpf_xdp_link_release,
9410         .dealloc = bpf_xdp_link_dealloc,
9411         .detach = bpf_xdp_link_detach,
9412         .show_fdinfo = bpf_xdp_link_show_fdinfo,
9413         .fill_link_info = bpf_xdp_link_fill_link_info,
9414         .update_prog = bpf_xdp_link_update,
9415 };
9416
9417 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
9418 {
9419         struct net *net = current->nsproxy->net_ns;
9420         struct bpf_link_primer link_primer;
9421         struct bpf_xdp_link *link;
9422         struct net_device *dev;
9423         int err, fd;
9424
9425         rtnl_lock();
9426         dev = dev_get_by_index(net, attr->link_create.target_ifindex);
9427         if (!dev) {
9428                 rtnl_unlock();
9429                 return -EINVAL;
9430         }
9431
9432         link = kzalloc(sizeof(*link), GFP_USER);
9433         if (!link) {
9434                 err = -ENOMEM;
9435                 goto unlock;
9436         }
9437
9438         bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
9439         link->dev = dev;
9440         link->flags = attr->link_create.flags;
9441
9442         err = bpf_link_prime(&link->link, &link_primer);
9443         if (err) {
9444                 kfree(link);
9445                 goto unlock;
9446         }
9447
9448         err = dev_xdp_attach_link(dev, NULL, link);
9449         rtnl_unlock();
9450
9451         if (err) {
9452                 link->dev = NULL;
9453                 bpf_link_cleanup(&link_primer);
9454                 goto out_put_dev;
9455         }
9456
9457         fd = bpf_link_settle(&link_primer);
9458         /* link itself doesn't hold dev's refcnt to not complicate shutdown */
9459         dev_put(dev);
9460         return fd;
9461
9462 unlock:
9463         rtnl_unlock();
9464
9465 out_put_dev:
9466         dev_put(dev);
9467         return err;
9468 }
9469
9470 /**
9471  *      dev_change_xdp_fd - set or clear a bpf program for a device rx path
9472  *      @dev: device
9473  *      @extack: netlink extended ack
9474  *      @fd: new program fd or negative value to clear
9475  *      @expected_fd: old program fd that userspace expects to replace or clear
9476  *      @flags: xdp-related flags
9477  *
9478  *      Set or clear a bpf program for a device
9479  */
9480 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
9481                       int fd, int expected_fd, u32 flags)
9482 {
9483         enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
9484         struct bpf_prog *new_prog = NULL, *old_prog = NULL;
9485         int err;
9486
9487         ASSERT_RTNL();
9488
9489         if (fd >= 0) {
9490                 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
9491                                                  mode != XDP_MODE_SKB);
9492                 if (IS_ERR(new_prog))
9493                         return PTR_ERR(new_prog);
9494         }
9495
9496         if (expected_fd >= 0) {
9497                 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
9498                                                  mode != XDP_MODE_SKB);
9499                 if (IS_ERR(old_prog)) {
9500                         err = PTR_ERR(old_prog);
9501                         old_prog = NULL;
9502                         goto err_out;
9503                 }
9504         }
9505
9506         err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
9507
9508 err_out:
9509         if (err && new_prog)
9510                 bpf_prog_put(new_prog);
9511         if (old_prog)
9512                 bpf_prog_put(old_prog);
9513         return err;
9514 }
9515
9516 /**
9517  *      dev_new_index   -       allocate an ifindex
9518  *      @net: the applicable net namespace
9519  *
9520  *      Returns a suitable unique value for a new device interface
9521  *      number.  The caller must hold the rtnl semaphore or the
9522  *      dev_base_lock to be sure it remains unique.
9523  */
9524 static int dev_new_index(struct net *net)
9525 {
9526         int ifindex = net->ifindex;
9527
9528         for (;;) {
9529                 if (++ifindex <= 0)
9530                         ifindex = 1;
9531                 if (!__dev_get_by_index(net, ifindex))
9532                         return net->ifindex = ifindex;
9533         }
9534 }
9535
9536 /* Delayed registration/unregisteration */
9537 LIST_HEAD(net_todo_list);
9538 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
9539
9540 static void net_set_todo(struct net_device *dev)
9541 {
9542         list_add_tail(&dev->todo_list, &net_todo_list);
9543         atomic_inc(&dev_net(dev)->dev_unreg_count);
9544 }
9545
9546 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
9547         struct net_device *upper, netdev_features_t features)
9548 {
9549         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9550         netdev_features_t feature;
9551         int feature_bit;
9552
9553         for_each_netdev_feature(upper_disables, feature_bit) {
9554                 feature = __NETIF_F_BIT(feature_bit);
9555                 if (!(upper->wanted_features & feature)
9556                     && (features & feature)) {
9557                         netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
9558                                    &feature, upper->name);
9559                         features &= ~feature;
9560                 }
9561         }
9562
9563         return features;
9564 }
9565
9566 static void netdev_sync_lower_features(struct net_device *upper,
9567         struct net_device *lower, netdev_features_t features)
9568 {
9569         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9570         netdev_features_t feature;
9571         int feature_bit;
9572
9573         for_each_netdev_feature(upper_disables, feature_bit) {
9574                 feature = __NETIF_F_BIT(feature_bit);
9575                 if (!(features & feature) && (lower->features & feature)) {
9576                         netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
9577                                    &feature, lower->name);
9578                         lower->wanted_features &= ~feature;
9579                         __netdev_update_features(lower);
9580
9581                         if (unlikely(lower->features & feature))
9582                                 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
9583                                             &feature, lower->name);
9584                         else
9585                                 netdev_features_change(lower);
9586                 }
9587         }
9588 }
9589
9590 static netdev_features_t netdev_fix_features(struct net_device *dev,
9591         netdev_features_t features)
9592 {
9593         /* Fix illegal checksum combinations */
9594         if ((features & NETIF_F_HW_CSUM) &&
9595             (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
9596                 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
9597                 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
9598         }
9599
9600         /* TSO requires that SG is present as well. */
9601         if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
9602                 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
9603                 features &= ~NETIF_F_ALL_TSO;
9604         }
9605
9606         if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
9607                                         !(features & NETIF_F_IP_CSUM)) {
9608                 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
9609                 features &= ~NETIF_F_TSO;
9610                 features &= ~NETIF_F_TSO_ECN;
9611         }
9612
9613         if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
9614                                          !(features & NETIF_F_IPV6_CSUM)) {
9615                 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
9616                 features &= ~NETIF_F_TSO6;
9617         }
9618
9619         /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
9620         if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
9621                 features &= ~NETIF_F_TSO_MANGLEID;
9622
9623         /* TSO ECN requires that TSO is present as well. */
9624         if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
9625                 features &= ~NETIF_F_TSO_ECN;
9626
9627         /* Software GSO depends on SG. */
9628         if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
9629                 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
9630                 features &= ~NETIF_F_GSO;
9631         }
9632
9633         /* GSO partial features require GSO partial be set */
9634         if ((features & dev->gso_partial_features) &&
9635             !(features & NETIF_F_GSO_PARTIAL)) {
9636                 netdev_dbg(dev,
9637                            "Dropping partially supported GSO features since no GSO partial.\n");
9638                 features &= ~dev->gso_partial_features;
9639         }
9640
9641         if (!(features & NETIF_F_RXCSUM)) {
9642                 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
9643                  * successfully merged by hardware must also have the
9644                  * checksum verified by hardware.  If the user does not
9645                  * want to enable RXCSUM, logically, we should disable GRO_HW.
9646                  */
9647                 if (features & NETIF_F_GRO_HW) {
9648                         netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
9649                         features &= ~NETIF_F_GRO_HW;
9650                 }
9651         }
9652
9653         /* LRO/HW-GRO features cannot be combined with RX-FCS */
9654         if (features & NETIF_F_RXFCS) {
9655                 if (features & NETIF_F_LRO) {
9656                         netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
9657                         features &= ~NETIF_F_LRO;
9658                 }
9659
9660                 if (features & NETIF_F_GRO_HW) {
9661                         netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
9662                         features &= ~NETIF_F_GRO_HW;
9663                 }
9664         }
9665
9666         if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) {
9667                 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n");
9668                 features &= ~NETIF_F_LRO;
9669         }
9670
9671         if (features & NETIF_F_HW_TLS_TX) {
9672                 bool ip_csum = (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) ==
9673                         (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
9674                 bool hw_csum = features & NETIF_F_HW_CSUM;
9675
9676                 if (!ip_csum && !hw_csum) {
9677                         netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
9678                         features &= ~NETIF_F_HW_TLS_TX;
9679                 }
9680         }
9681
9682         if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
9683                 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
9684                 features &= ~NETIF_F_HW_TLS_RX;
9685         }
9686
9687         return features;
9688 }
9689
9690 int __netdev_update_features(struct net_device *dev)
9691 {
9692         struct net_device *upper, *lower;
9693         netdev_features_t features;
9694         struct list_head *iter;
9695         int err = -1;
9696
9697         ASSERT_RTNL();
9698
9699         features = netdev_get_wanted_features(dev);
9700
9701         if (dev->netdev_ops->ndo_fix_features)
9702                 features = dev->netdev_ops->ndo_fix_features(dev, features);
9703
9704         /* driver might be less strict about feature dependencies */
9705         features = netdev_fix_features(dev, features);
9706
9707         /* some features can't be enabled if they're off on an upper device */
9708         netdev_for_each_upper_dev_rcu(dev, upper, iter)
9709                 features = netdev_sync_upper_features(dev, upper, features);
9710
9711         if (dev->features == features)
9712                 goto sync_lower;
9713
9714         netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
9715                 &dev->features, &features);
9716
9717         if (dev->netdev_ops->ndo_set_features)
9718                 err = dev->netdev_ops->ndo_set_features(dev, features);
9719         else
9720                 err = 0;
9721
9722         if (unlikely(err < 0)) {
9723                 netdev_err(dev,
9724                         "set_features() failed (%d); wanted %pNF, left %pNF\n",
9725                         err, &features, &dev->features);
9726                 /* return non-0 since some features might have changed and
9727                  * it's better to fire a spurious notification than miss it
9728                  */
9729                 return -1;
9730         }
9731
9732 sync_lower:
9733         /* some features must be disabled on lower devices when disabled
9734          * on an upper device (think: bonding master or bridge)
9735          */
9736         netdev_for_each_lower_dev(dev, lower, iter)
9737                 netdev_sync_lower_features(dev, lower, features);
9738
9739         if (!err) {
9740                 netdev_features_t diff = features ^ dev->features;
9741
9742                 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
9743                         /* udp_tunnel_{get,drop}_rx_info both need
9744                          * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
9745                          * device, or they won't do anything.
9746                          * Thus we need to update dev->features
9747                          * *before* calling udp_tunnel_get_rx_info,
9748                          * but *after* calling udp_tunnel_drop_rx_info.
9749                          */
9750                         if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
9751                                 dev->features = features;
9752                                 udp_tunnel_get_rx_info(dev);
9753                         } else {
9754                                 udp_tunnel_drop_rx_info(dev);
9755                         }
9756                 }
9757
9758                 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
9759                         if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
9760                                 dev->features = features;
9761                                 err |= vlan_get_rx_ctag_filter_info(dev);
9762                         } else {
9763                                 vlan_drop_rx_ctag_filter_info(dev);
9764                         }
9765                 }
9766
9767                 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
9768                         if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
9769                                 dev->features = features;
9770                                 err |= vlan_get_rx_stag_filter_info(dev);
9771                         } else {
9772                                 vlan_drop_rx_stag_filter_info(dev);
9773                         }
9774                 }
9775
9776                 dev->features = features;
9777         }
9778
9779         return err < 0 ? 0 : 1;
9780 }
9781
9782 /**
9783  *      netdev_update_features - recalculate device features
9784  *      @dev: the device to check
9785  *
9786  *      Recalculate dev->features set and send notifications if it
9787  *      has changed. Should be called after driver or hardware dependent
9788  *      conditions might have changed that influence the features.
9789  */
9790 void netdev_update_features(struct net_device *dev)
9791 {
9792         if (__netdev_update_features(dev))
9793                 netdev_features_change(dev);
9794 }
9795 EXPORT_SYMBOL(netdev_update_features);
9796
9797 /**
9798  *      netdev_change_features - recalculate device features
9799  *      @dev: the device to check
9800  *
9801  *      Recalculate dev->features set and send notifications even
9802  *      if they have not changed. Should be called instead of
9803  *      netdev_update_features() if also dev->vlan_features might
9804  *      have changed to allow the changes to be propagated to stacked
9805  *      VLAN devices.
9806  */
9807 void netdev_change_features(struct net_device *dev)
9808 {
9809         __netdev_update_features(dev);
9810         netdev_features_change(dev);
9811 }
9812 EXPORT_SYMBOL(netdev_change_features);
9813
9814 /**
9815  *      netif_stacked_transfer_operstate -      transfer operstate
9816  *      @rootdev: the root or lower level device to transfer state from
9817  *      @dev: the device to transfer operstate to
9818  *
9819  *      Transfer operational state from root to device. This is normally
9820  *      called when a stacking relationship exists between the root
9821  *      device and the device(a leaf device).
9822  */
9823 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
9824                                         struct net_device *dev)
9825 {
9826         if (rootdev->operstate == IF_OPER_DORMANT)
9827                 netif_dormant_on(dev);
9828         else
9829                 netif_dormant_off(dev);
9830
9831         if (rootdev->operstate == IF_OPER_TESTING)
9832                 netif_testing_on(dev);
9833         else
9834                 netif_testing_off(dev);
9835
9836         if (netif_carrier_ok(rootdev))
9837                 netif_carrier_on(dev);
9838         else
9839                 netif_carrier_off(dev);
9840 }
9841 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
9842
9843 static int netif_alloc_rx_queues(struct net_device *dev)
9844 {
9845         unsigned int i, count = dev->num_rx_queues;
9846         struct netdev_rx_queue *rx;
9847         size_t sz = count * sizeof(*rx);
9848         int err = 0;
9849
9850         BUG_ON(count < 1);
9851
9852         rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
9853         if (!rx)
9854                 return -ENOMEM;
9855
9856         dev->_rx = rx;
9857
9858         for (i = 0; i < count; i++) {
9859                 rx[i].dev = dev;
9860
9861                 /* XDP RX-queue setup */
9862                 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
9863                 if (err < 0)
9864                         goto err_rxq_info;
9865         }
9866         return 0;
9867
9868 err_rxq_info:
9869         /* Rollback successful reg's and free other resources */
9870         while (i--)
9871                 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
9872         kvfree(dev->_rx);
9873         dev->_rx = NULL;
9874         return err;
9875 }
9876
9877 static void netif_free_rx_queues(struct net_device *dev)
9878 {
9879         unsigned int i, count = dev->num_rx_queues;
9880
9881         /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
9882         if (!dev->_rx)
9883                 return;
9884
9885         for (i = 0; i < count; i++)
9886                 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
9887
9888         kvfree(dev->_rx);
9889 }
9890
9891 static void netdev_init_one_queue(struct net_device *dev,
9892                                   struct netdev_queue *queue, void *_unused)
9893 {
9894         /* Initialize queue lock */
9895         spin_lock_init(&queue->_xmit_lock);
9896         netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
9897         queue->xmit_lock_owner = -1;
9898         netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
9899         queue->dev = dev;
9900 #ifdef CONFIG_BQL
9901         dql_init(&queue->dql, HZ);
9902 #endif
9903 }
9904
9905 static void netif_free_tx_queues(struct net_device *dev)
9906 {
9907         kvfree(dev->_tx);
9908 }
9909
9910 static int netif_alloc_netdev_queues(struct net_device *dev)
9911 {
9912         unsigned int count = dev->num_tx_queues;
9913         struct netdev_queue *tx;
9914         size_t sz = count * sizeof(*tx);
9915
9916         if (count < 1 || count > 0xffff)
9917                 return -EINVAL;
9918
9919         tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
9920         if (!tx)
9921                 return -ENOMEM;
9922
9923         dev->_tx = tx;
9924
9925         netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
9926         spin_lock_init(&dev->tx_global_lock);
9927
9928         return 0;
9929 }
9930
9931 void netif_tx_stop_all_queues(struct net_device *dev)
9932 {
9933         unsigned int i;
9934
9935         for (i = 0; i < dev->num_tx_queues; i++) {
9936                 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
9937
9938                 netif_tx_stop_queue(txq);
9939         }
9940 }
9941 EXPORT_SYMBOL(netif_tx_stop_all_queues);
9942
9943 /**
9944  * register_netdevice() - register a network device
9945  * @dev: device to register
9946  *
9947  * Take a prepared network device structure and make it externally accessible.
9948  * A %NETDEV_REGISTER message is sent to the netdev notifier chain.
9949  * Callers must hold the rtnl lock - you may want register_netdev()
9950  * instead of this.
9951  */
9952 int register_netdevice(struct net_device *dev)
9953 {
9954         int ret;
9955         struct net *net = dev_net(dev);
9956
9957         BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
9958                      NETDEV_FEATURE_COUNT);
9959         BUG_ON(dev_boot_phase);
9960         ASSERT_RTNL();
9961
9962         might_sleep();
9963
9964         /* When net_device's are persistent, this will be fatal. */
9965         BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
9966         BUG_ON(!net);
9967
9968         ret = ethtool_check_ops(dev->ethtool_ops);
9969         if (ret)
9970                 return ret;
9971
9972         spin_lock_init(&dev->addr_list_lock);
9973         netdev_set_addr_lockdep_class(dev);
9974
9975         ret = dev_get_valid_name(net, dev, dev->name);
9976         if (ret < 0)
9977                 goto out;
9978
9979         ret = -ENOMEM;
9980         dev->name_node = netdev_name_node_head_alloc(dev);
9981         if (!dev->name_node)
9982                 goto out;
9983
9984         /* Init, if this function is available */
9985         if (dev->netdev_ops->ndo_init) {
9986                 ret = dev->netdev_ops->ndo_init(dev);
9987                 if (ret) {
9988                         if (ret > 0)
9989                                 ret = -EIO;
9990                         goto err_free_name;
9991                 }
9992         }
9993
9994         if (((dev->hw_features | dev->features) &
9995              NETIF_F_HW_VLAN_CTAG_FILTER) &&
9996             (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
9997              !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
9998                 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
9999                 ret = -EINVAL;
10000                 goto err_uninit;
10001         }
10002
10003         ret = -EBUSY;
10004         if (!dev->ifindex)
10005                 dev->ifindex = dev_new_index(net);
10006         else if (__dev_get_by_index(net, dev->ifindex))
10007                 goto err_uninit;
10008
10009         /* Transfer changeable features to wanted_features and enable
10010          * software offloads (GSO and GRO).
10011          */
10012         dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
10013         dev->features |= NETIF_F_SOFT_FEATURES;
10014
10015         if (dev->udp_tunnel_nic_info) {
10016                 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10017                 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10018         }
10019
10020         dev->wanted_features = dev->features & dev->hw_features;
10021
10022         if (!(dev->flags & IFF_LOOPBACK))
10023                 dev->hw_features |= NETIF_F_NOCACHE_COPY;
10024
10025         /* If IPv4 TCP segmentation offload is supported we should also
10026          * allow the device to enable segmenting the frame with the option
10027          * of ignoring a static IP ID value.  This doesn't enable the
10028          * feature itself but allows the user to enable it later.
10029          */
10030         if (dev->hw_features & NETIF_F_TSO)
10031                 dev->hw_features |= NETIF_F_TSO_MANGLEID;
10032         if (dev->vlan_features & NETIF_F_TSO)
10033                 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
10034         if (dev->mpls_features & NETIF_F_TSO)
10035                 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
10036         if (dev->hw_enc_features & NETIF_F_TSO)
10037                 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
10038
10039         /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
10040          */
10041         dev->vlan_features |= NETIF_F_HIGHDMA;
10042
10043         /* Make NETIF_F_SG inheritable to tunnel devices.
10044          */
10045         dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
10046
10047         /* Make NETIF_F_SG inheritable to MPLS.
10048          */
10049         dev->mpls_features |= NETIF_F_SG;
10050
10051         ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
10052         ret = notifier_to_errno(ret);
10053         if (ret)
10054                 goto err_uninit;
10055
10056         ret = netdev_register_kobject(dev);
10057         write_lock(&dev_base_lock);
10058         dev->reg_state = ret ? NETREG_UNREGISTERED : NETREG_REGISTERED;
10059         write_unlock(&dev_base_lock);
10060         if (ret)
10061                 goto err_uninit_notify;
10062
10063         __netdev_update_features(dev);
10064
10065         /*
10066          *      Default initial state at registry is that the
10067          *      device is present.
10068          */
10069
10070         set_bit(__LINK_STATE_PRESENT, &dev->state);
10071
10072         linkwatch_init_dev(dev);
10073
10074         dev_init_scheduler(dev);
10075
10076         netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL);
10077         list_netdevice(dev);
10078
10079         add_device_randomness(dev->dev_addr, dev->addr_len);
10080
10081         /* If the device has permanent device address, driver should
10082          * set dev_addr and also addr_assign_type should be set to
10083          * NET_ADDR_PERM (default value).
10084          */
10085         if (dev->addr_assign_type == NET_ADDR_PERM)
10086                 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10087
10088         /* Notify protocols, that a new device appeared. */
10089         ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
10090         ret = notifier_to_errno(ret);
10091         if (ret) {
10092                 /* Expect explicit free_netdev() on failure */
10093                 dev->needs_free_netdev = false;
10094                 unregister_netdevice_queue(dev, NULL);
10095                 goto out;
10096         }
10097         /*
10098          *      Prevent userspace races by waiting until the network
10099          *      device is fully setup before sending notifications.
10100          */
10101         if (!dev->rtnl_link_ops ||
10102             dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10103                 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
10104
10105 out:
10106         return ret;
10107
10108 err_uninit_notify:
10109         call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
10110 err_uninit:
10111         if (dev->netdev_ops->ndo_uninit)
10112                 dev->netdev_ops->ndo_uninit(dev);
10113         if (dev->priv_destructor)
10114                 dev->priv_destructor(dev);
10115 err_free_name:
10116         netdev_name_node_free(dev->name_node);
10117         goto out;
10118 }
10119 EXPORT_SYMBOL(register_netdevice);
10120
10121 /**
10122  *      init_dummy_netdev       - init a dummy network device for NAPI
10123  *      @dev: device to init
10124  *
10125  *      This takes a network device structure and initialize the minimum
10126  *      amount of fields so it can be used to schedule NAPI polls without
10127  *      registering a full blown interface. This is to be used by drivers
10128  *      that need to tie several hardware interfaces to a single NAPI
10129  *      poll scheduler due to HW limitations.
10130  */
10131 int init_dummy_netdev(struct net_device *dev)
10132 {
10133         /* Clear everything. Note we don't initialize spinlocks
10134          * are they aren't supposed to be taken by any of the
10135          * NAPI code and this dummy netdev is supposed to be
10136          * only ever used for NAPI polls
10137          */
10138         memset(dev, 0, sizeof(struct net_device));
10139
10140         /* make sure we BUG if trying to hit standard
10141          * register/unregister code path
10142          */
10143         dev->reg_state = NETREG_DUMMY;
10144
10145         /* NAPI wants this */
10146         INIT_LIST_HEAD(&dev->napi_list);
10147
10148         /* a dummy interface is started by default */
10149         set_bit(__LINK_STATE_PRESENT, &dev->state);
10150         set_bit(__LINK_STATE_START, &dev->state);
10151
10152         /* napi_busy_loop stats accounting wants this */
10153         dev_net_set(dev, &init_net);
10154
10155         /* Note : We dont allocate pcpu_refcnt for dummy devices,
10156          * because users of this 'device' dont need to change
10157          * its refcount.
10158          */
10159
10160         return 0;
10161 }
10162 EXPORT_SYMBOL_GPL(init_dummy_netdev);
10163
10164
10165 /**
10166  *      register_netdev - register a network device
10167  *      @dev: device to register
10168  *
10169  *      Take a completed network device structure and add it to the kernel
10170  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10171  *      chain. 0 is returned on success. A negative errno code is returned
10172  *      on a failure to set up the device, or if the name is a duplicate.
10173  *
10174  *      This is a wrapper around register_netdevice that takes the rtnl semaphore
10175  *      and expands the device name if you passed a format string to
10176  *      alloc_netdev.
10177  */
10178 int register_netdev(struct net_device *dev)
10179 {
10180         int err;
10181
10182         if (rtnl_lock_killable())
10183                 return -EINTR;
10184         err = register_netdevice(dev);
10185         rtnl_unlock();
10186         return err;
10187 }
10188 EXPORT_SYMBOL(register_netdev);
10189
10190 int netdev_refcnt_read(const struct net_device *dev)
10191 {
10192 #ifdef CONFIG_PCPU_DEV_REFCNT
10193         int i, refcnt = 0;
10194
10195         for_each_possible_cpu(i)
10196                 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10197         return refcnt;
10198 #else
10199         return refcount_read(&dev->dev_refcnt);
10200 #endif
10201 }
10202 EXPORT_SYMBOL(netdev_refcnt_read);
10203
10204 int netdev_unregister_timeout_secs __read_mostly = 10;
10205
10206 #define WAIT_REFS_MIN_MSECS 1
10207 #define WAIT_REFS_MAX_MSECS 250
10208 /**
10209  * netdev_wait_allrefs_any - wait until all references are gone.
10210  * @list: list of net_devices to wait on
10211  *
10212  * This is called when unregistering network devices.
10213  *
10214  * Any protocol or device that holds a reference should register
10215  * for netdevice notification, and cleanup and put back the
10216  * reference if they receive an UNREGISTER event.
10217  * We can get stuck here if buggy protocols don't correctly
10218  * call dev_put.
10219  */
10220 static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
10221 {
10222         unsigned long rebroadcast_time, warning_time;
10223         struct net_device *dev;
10224         int wait = 0;
10225
10226         rebroadcast_time = warning_time = jiffies;
10227
10228         list_for_each_entry(dev, list, todo_list)
10229                 if (netdev_refcnt_read(dev) == 1)
10230                         return dev;
10231
10232         while (true) {
10233                 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
10234                         rtnl_lock();
10235
10236                         /* Rebroadcast unregister notification */
10237                         list_for_each_entry(dev, list, todo_list)
10238                                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10239
10240                         __rtnl_unlock();
10241                         rcu_barrier();
10242                         rtnl_lock();
10243
10244                         list_for_each_entry(dev, list, todo_list)
10245                                 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
10246                                              &dev->state)) {
10247                                         /* We must not have linkwatch events
10248                                          * pending on unregister. If this
10249                                          * happens, we simply run the queue
10250                                          * unscheduled, resulting in a noop
10251                                          * for this device.
10252                                          */
10253                                         linkwatch_run_queue();
10254                                         break;
10255                                 }
10256
10257                         __rtnl_unlock();
10258
10259                         rebroadcast_time = jiffies;
10260                 }
10261
10262                 if (!wait) {
10263                         rcu_barrier();
10264                         wait = WAIT_REFS_MIN_MSECS;
10265                 } else {
10266                         msleep(wait);
10267                         wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10268                 }
10269
10270                 list_for_each_entry(dev, list, todo_list)
10271                         if (netdev_refcnt_read(dev) == 1)
10272                                 return dev;
10273
10274                 if (time_after(jiffies, warning_time +
10275                                READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
10276                         list_for_each_entry(dev, list, todo_list) {
10277                                 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10278                                          dev->name, netdev_refcnt_read(dev));
10279                                 ref_tracker_dir_print(&dev->refcnt_tracker, 10);
10280                         }
10281
10282                         warning_time = jiffies;
10283                 }
10284         }
10285 }
10286
10287 /* The sequence is:
10288  *
10289  *      rtnl_lock();
10290  *      ...
10291  *      register_netdevice(x1);
10292  *      register_netdevice(x2);
10293  *      ...
10294  *      unregister_netdevice(y1);
10295  *      unregister_netdevice(y2);
10296  *      ...
10297  *      rtnl_unlock();
10298  *      free_netdev(y1);
10299  *      free_netdev(y2);
10300  *
10301  * We are invoked by rtnl_unlock().
10302  * This allows us to deal with problems:
10303  * 1) We can delete sysfs objects which invoke hotplug
10304  *    without deadlocking with linkwatch via keventd.
10305  * 2) Since we run with the RTNL semaphore not held, we can sleep
10306  *    safely in order to wait for the netdev refcnt to drop to zero.
10307  *
10308  * We must not return until all unregister events added during
10309  * the interval the lock was held have been completed.
10310  */
10311 void netdev_run_todo(void)
10312 {
10313         struct net_device *dev, *tmp;
10314         struct list_head list;
10315 #ifdef CONFIG_LOCKDEP
10316         struct list_head unlink_list;
10317
10318         list_replace_init(&net_unlink_list, &unlink_list);
10319
10320         while (!list_empty(&unlink_list)) {
10321                 struct net_device *dev = list_first_entry(&unlink_list,
10322                                                           struct net_device,
10323                                                           unlink_list);
10324                 list_del_init(&dev->unlink_list);
10325                 dev->nested_level = dev->lower_level - 1;
10326         }
10327 #endif
10328
10329         /* Snapshot list, allow later requests */
10330         list_replace_init(&net_todo_list, &list);
10331
10332         __rtnl_unlock();
10333
10334         /* Wait for rcu callbacks to finish before next phase */
10335         if (!list_empty(&list))
10336                 rcu_barrier();
10337
10338         list_for_each_entry_safe(dev, tmp, &list, todo_list) {
10339                 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
10340                         netdev_WARN(dev, "run_todo but not unregistering\n");
10341                         list_del(&dev->todo_list);
10342                         continue;
10343                 }
10344
10345                 write_lock(&dev_base_lock);
10346                 dev->reg_state = NETREG_UNREGISTERED;
10347                 write_unlock(&dev_base_lock);
10348                 linkwatch_forget_dev(dev);
10349         }
10350
10351         while (!list_empty(&list)) {
10352                 dev = netdev_wait_allrefs_any(&list);
10353                 list_del(&dev->todo_list);
10354
10355                 /* paranoia */
10356                 BUG_ON(netdev_refcnt_read(dev) != 1);
10357                 BUG_ON(!list_empty(&dev->ptype_all));
10358                 BUG_ON(!list_empty(&dev->ptype_specific));
10359                 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10360                 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
10361
10362                 if (dev->priv_destructor)
10363                         dev->priv_destructor(dev);
10364                 if (dev->needs_free_netdev)
10365                         free_netdev(dev);
10366
10367                 if (atomic_dec_and_test(&dev_net(dev)->dev_unreg_count))
10368                         wake_up(&netdev_unregistering_wq);
10369
10370                 /* Free network device */
10371                 kobject_put(&dev->dev.kobj);
10372         }
10373 }
10374
10375 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
10376  * all the same fields in the same order as net_device_stats, with only
10377  * the type differing, but rtnl_link_stats64 may have additional fields
10378  * at the end for newer counters.
10379  */
10380 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
10381                              const struct net_device_stats *netdev_stats)
10382 {
10383         size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t);
10384         const atomic_long_t *src = (atomic_long_t *)netdev_stats;
10385         u64 *dst = (u64 *)stats64;
10386
10387         BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
10388         for (i = 0; i < n; i++)
10389                 dst[i] = (unsigned long)atomic_long_read(&src[i]);
10390         /* zero out counters that only exist in rtnl_link_stats64 */
10391         memset((char *)stats64 + n * sizeof(u64), 0,
10392                sizeof(*stats64) - n * sizeof(u64));
10393 }
10394 EXPORT_SYMBOL(netdev_stats_to_stats64);
10395
10396 struct net_device_core_stats __percpu *netdev_core_stats_alloc(struct net_device *dev)
10397 {
10398         struct net_device_core_stats __percpu *p;
10399
10400         p = alloc_percpu_gfp(struct net_device_core_stats,
10401                              GFP_ATOMIC | __GFP_NOWARN);
10402
10403         if (p && cmpxchg(&dev->core_stats, NULL, p))
10404                 free_percpu(p);
10405
10406         /* This READ_ONCE() pairs with the cmpxchg() above */
10407         return READ_ONCE(dev->core_stats);
10408 }
10409 EXPORT_SYMBOL(netdev_core_stats_alloc);
10410
10411 /**
10412  *      dev_get_stats   - get network device statistics
10413  *      @dev: device to get statistics from
10414  *      @storage: place to store stats
10415  *
10416  *      Get network statistics from device. Return @storage.
10417  *      The device driver may provide its own method by setting
10418  *      dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
10419  *      otherwise the internal statistics structure is used.
10420  */
10421 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
10422                                         struct rtnl_link_stats64 *storage)
10423 {
10424         const struct net_device_ops *ops = dev->netdev_ops;
10425         const struct net_device_core_stats __percpu *p;
10426
10427         if (ops->ndo_get_stats64) {
10428                 memset(storage, 0, sizeof(*storage));
10429                 ops->ndo_get_stats64(dev, storage);
10430         } else if (ops->ndo_get_stats) {
10431                 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
10432         } else {
10433                 netdev_stats_to_stats64(storage, &dev->stats);
10434         }
10435
10436         /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
10437         p = READ_ONCE(dev->core_stats);
10438         if (p) {
10439                 const struct net_device_core_stats *core_stats;
10440                 int i;
10441
10442                 for_each_possible_cpu(i) {
10443                         core_stats = per_cpu_ptr(p, i);
10444                         storage->rx_dropped += READ_ONCE(core_stats->rx_dropped);
10445                         storage->tx_dropped += READ_ONCE(core_stats->tx_dropped);
10446                         storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler);
10447                         storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped);
10448                 }
10449         }
10450         return storage;
10451 }
10452 EXPORT_SYMBOL(dev_get_stats);
10453
10454 /**
10455  *      dev_fetch_sw_netstats - get per-cpu network device statistics
10456  *      @s: place to store stats
10457  *      @netstats: per-cpu network stats to read from
10458  *
10459  *      Read per-cpu network statistics and populate the related fields in @s.
10460  */
10461 void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
10462                            const struct pcpu_sw_netstats __percpu *netstats)
10463 {
10464         int cpu;
10465
10466         for_each_possible_cpu(cpu) {
10467                 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
10468                 const struct pcpu_sw_netstats *stats;
10469                 unsigned int start;
10470
10471                 stats = per_cpu_ptr(netstats, cpu);
10472                 do {
10473                         start = u64_stats_fetch_begin(&stats->syncp);
10474                         rx_packets = u64_stats_read(&stats->rx_packets);
10475                         rx_bytes   = u64_stats_read(&stats->rx_bytes);
10476                         tx_packets = u64_stats_read(&stats->tx_packets);
10477                         tx_bytes   = u64_stats_read(&stats->tx_bytes);
10478                 } while (u64_stats_fetch_retry(&stats->syncp, start));
10479
10480                 s->rx_packets += rx_packets;
10481                 s->rx_bytes   += rx_bytes;
10482                 s->tx_packets += tx_packets;
10483                 s->tx_bytes   += tx_bytes;
10484         }
10485 }
10486 EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
10487
10488 /**
10489  *      dev_get_tstats64 - ndo_get_stats64 implementation
10490  *      @dev: device to get statistics from
10491  *      @s: place to store stats
10492  *
10493  *      Populate @s from dev->stats and dev->tstats. Can be used as
10494  *      ndo_get_stats64() callback.
10495  */
10496 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
10497 {
10498         netdev_stats_to_stats64(s, &dev->stats);
10499         dev_fetch_sw_netstats(s, dev->tstats);
10500 }
10501 EXPORT_SYMBOL_GPL(dev_get_tstats64);
10502
10503 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
10504 {
10505         struct netdev_queue *queue = dev_ingress_queue(dev);
10506
10507 #ifdef CONFIG_NET_CLS_ACT
10508         if (queue)
10509                 return queue;
10510         queue = kzalloc(sizeof(*queue), GFP_KERNEL);
10511         if (!queue)
10512                 return NULL;
10513         netdev_init_one_queue(dev, queue, NULL);
10514         RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
10515         queue->qdisc_sleeping = &noop_qdisc;
10516         rcu_assign_pointer(dev->ingress_queue, queue);
10517 #endif
10518         return queue;
10519 }
10520
10521 static const struct ethtool_ops default_ethtool_ops;
10522
10523 void netdev_set_default_ethtool_ops(struct net_device *dev,
10524                                     const struct ethtool_ops *ops)
10525 {
10526         if (dev->ethtool_ops == &default_ethtool_ops)
10527                 dev->ethtool_ops = ops;
10528 }
10529 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
10530
10531 /**
10532  * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
10533  * @dev: netdev to enable the IRQ coalescing on
10534  *
10535  * Sets a conservative default for SW IRQ coalescing. Users can use
10536  * sysfs attributes to override the default values.
10537  */
10538 void netdev_sw_irq_coalesce_default_on(struct net_device *dev)
10539 {
10540         WARN_ON(dev->reg_state == NETREG_REGISTERED);
10541
10542         dev->gro_flush_timeout = 20000;
10543         dev->napi_defer_hard_irqs = 1;
10544 }
10545 EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on);
10546
10547 void netdev_freemem(struct net_device *dev)
10548 {
10549         char *addr = (char *)dev - dev->padded;
10550
10551         kvfree(addr);
10552 }
10553
10554 /**
10555  * alloc_netdev_mqs - allocate network device
10556  * @sizeof_priv: size of private data to allocate space for
10557  * @name: device name format string
10558  * @name_assign_type: origin of device name
10559  * @setup: callback to initialize device
10560  * @txqs: the number of TX subqueues to allocate
10561  * @rxqs: the number of RX subqueues to allocate
10562  *
10563  * Allocates a struct net_device with private data area for driver use
10564  * and performs basic initialization.  Also allocates subqueue structs
10565  * for each queue on the device.
10566  */
10567 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
10568                 unsigned char name_assign_type,
10569                 void (*setup)(struct net_device *),
10570                 unsigned int txqs, unsigned int rxqs)
10571 {
10572         struct net_device *dev;
10573         unsigned int alloc_size;
10574         struct net_device *p;
10575
10576         BUG_ON(strlen(name) >= sizeof(dev->name));
10577
10578         if (txqs < 1) {
10579                 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
10580                 return NULL;
10581         }
10582
10583         if (rxqs < 1) {
10584                 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
10585                 return NULL;
10586         }
10587
10588         alloc_size = sizeof(struct net_device);
10589         if (sizeof_priv) {
10590                 /* ensure 32-byte alignment of private area */
10591                 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
10592                 alloc_size += sizeof_priv;
10593         }
10594         /* ensure 32-byte alignment of whole construct */
10595         alloc_size += NETDEV_ALIGN - 1;
10596
10597         p = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
10598         if (!p)
10599                 return NULL;
10600
10601         dev = PTR_ALIGN(p, NETDEV_ALIGN);
10602         dev->padded = (char *)dev - (char *)p;
10603
10604         ref_tracker_dir_init(&dev->refcnt_tracker, 128);
10605 #ifdef CONFIG_PCPU_DEV_REFCNT
10606         dev->pcpu_refcnt = alloc_percpu(int);
10607         if (!dev->pcpu_refcnt)
10608                 goto free_dev;
10609         __dev_hold(dev);
10610 #else
10611         refcount_set(&dev->dev_refcnt, 1);
10612 #endif
10613
10614         if (dev_addr_init(dev))
10615                 goto free_pcpu;
10616
10617         dev_mc_init(dev);
10618         dev_uc_init(dev);
10619
10620         dev_net_set(dev, &init_net);
10621
10622         dev->gso_max_size = GSO_LEGACY_MAX_SIZE;
10623         dev->gso_max_segs = GSO_MAX_SEGS;
10624         dev->gro_max_size = GRO_LEGACY_MAX_SIZE;
10625         dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE;
10626         dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE;
10627         dev->tso_max_size = TSO_LEGACY_MAX_SIZE;
10628         dev->tso_max_segs = TSO_MAX_SEGS;
10629         dev->upper_level = 1;
10630         dev->lower_level = 1;
10631 #ifdef CONFIG_LOCKDEP
10632         dev->nested_level = 0;
10633         INIT_LIST_HEAD(&dev->unlink_list);
10634 #endif
10635
10636         INIT_LIST_HEAD(&dev->napi_list);
10637         INIT_LIST_HEAD(&dev->unreg_list);
10638         INIT_LIST_HEAD(&dev->close_list);
10639         INIT_LIST_HEAD(&dev->link_watch_list);
10640         INIT_LIST_HEAD(&dev->adj_list.upper);
10641         INIT_LIST_HEAD(&dev->adj_list.lower);
10642         INIT_LIST_HEAD(&dev->ptype_all);
10643         INIT_LIST_HEAD(&dev->ptype_specific);
10644         INIT_LIST_HEAD(&dev->net_notifier_list);
10645 #ifdef CONFIG_NET_SCHED
10646         hash_init(dev->qdisc_hash);
10647 #endif
10648         dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
10649         setup(dev);
10650
10651         if (!dev->tx_queue_len) {
10652                 dev->priv_flags |= IFF_NO_QUEUE;
10653                 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
10654         }
10655
10656         dev->num_tx_queues = txqs;
10657         dev->real_num_tx_queues = txqs;
10658         if (netif_alloc_netdev_queues(dev))
10659                 goto free_all;
10660
10661         dev->num_rx_queues = rxqs;
10662         dev->real_num_rx_queues = rxqs;
10663         if (netif_alloc_rx_queues(dev))
10664                 goto free_all;
10665
10666         strcpy(dev->name, name);
10667         dev->name_assign_type = name_assign_type;
10668         dev->group = INIT_NETDEV_GROUP;
10669         if (!dev->ethtool_ops)
10670                 dev->ethtool_ops = &default_ethtool_ops;
10671
10672         nf_hook_netdev_init(dev);
10673
10674         return dev;
10675
10676 free_all:
10677         free_netdev(dev);
10678         return NULL;
10679
10680 free_pcpu:
10681 #ifdef CONFIG_PCPU_DEV_REFCNT
10682         free_percpu(dev->pcpu_refcnt);
10683 free_dev:
10684 #endif
10685         netdev_freemem(dev);
10686         return NULL;
10687 }
10688 EXPORT_SYMBOL(alloc_netdev_mqs);
10689
10690 /**
10691  * free_netdev - free network device
10692  * @dev: device
10693  *
10694  * This function does the last stage of destroying an allocated device
10695  * interface. The reference to the device object is released. If this
10696  * is the last reference then it will be freed.Must be called in process
10697  * context.
10698  */
10699 void free_netdev(struct net_device *dev)
10700 {
10701         struct napi_struct *p, *n;
10702
10703         might_sleep();
10704
10705         /* When called immediately after register_netdevice() failed the unwind
10706          * handling may still be dismantling the device. Handle that case by
10707          * deferring the free.
10708          */
10709         if (dev->reg_state == NETREG_UNREGISTERING) {
10710                 ASSERT_RTNL();
10711                 dev->needs_free_netdev = true;
10712                 return;
10713         }
10714
10715         netif_free_tx_queues(dev);
10716         netif_free_rx_queues(dev);
10717
10718         kfree(rcu_dereference_protected(dev->ingress_queue, 1));
10719
10720         /* Flush device addresses */
10721         dev_addr_flush(dev);
10722
10723         list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
10724                 netif_napi_del(p);
10725
10726         ref_tracker_dir_exit(&dev->refcnt_tracker);
10727 #ifdef CONFIG_PCPU_DEV_REFCNT
10728         free_percpu(dev->pcpu_refcnt);
10729         dev->pcpu_refcnt = NULL;
10730 #endif
10731         free_percpu(dev->core_stats);
10732         dev->core_stats = NULL;
10733         free_percpu(dev->xdp_bulkq);
10734         dev->xdp_bulkq = NULL;
10735
10736         /*  Compatibility with error handling in drivers */
10737         if (dev->reg_state == NETREG_UNINITIALIZED) {
10738                 netdev_freemem(dev);
10739                 return;
10740         }
10741
10742         BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
10743         dev->reg_state = NETREG_RELEASED;
10744
10745         /* will free via device release */
10746         put_device(&dev->dev);
10747 }
10748 EXPORT_SYMBOL(free_netdev);
10749
10750 /**
10751  *      synchronize_net -  Synchronize with packet receive processing
10752  *
10753  *      Wait for packets currently being received to be done.
10754  *      Does not block later packets from starting.
10755  */
10756 void synchronize_net(void)
10757 {
10758         might_sleep();
10759         if (rtnl_is_locked())
10760                 synchronize_rcu_expedited();
10761         else
10762                 synchronize_rcu();
10763 }
10764 EXPORT_SYMBOL(synchronize_net);
10765
10766 /**
10767  *      unregister_netdevice_queue - remove device from the kernel
10768  *      @dev: device
10769  *      @head: list
10770  *
10771  *      This function shuts down a device interface and removes it
10772  *      from the kernel tables.
10773  *      If head not NULL, device is queued to be unregistered later.
10774  *
10775  *      Callers must hold the rtnl semaphore.  You may want
10776  *      unregister_netdev() instead of this.
10777  */
10778
10779 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
10780 {
10781         ASSERT_RTNL();
10782
10783         if (head) {
10784                 list_move_tail(&dev->unreg_list, head);
10785         } else {
10786                 LIST_HEAD(single);
10787
10788                 list_add(&dev->unreg_list, &single);
10789                 unregister_netdevice_many(&single);
10790         }
10791 }
10792 EXPORT_SYMBOL(unregister_netdevice_queue);
10793
10794 void unregister_netdevice_many_notify(struct list_head *head,
10795                                       u32 portid, const struct nlmsghdr *nlh)
10796 {
10797         struct net_device *dev, *tmp;
10798         LIST_HEAD(close_head);
10799
10800         BUG_ON(dev_boot_phase);
10801         ASSERT_RTNL();
10802
10803         if (list_empty(head))
10804                 return;
10805
10806         list_for_each_entry_safe(dev, tmp, head, unreg_list) {
10807                 /* Some devices call without registering
10808                  * for initialization unwind. Remove those
10809                  * devices and proceed with the remaining.
10810                  */
10811                 if (dev->reg_state == NETREG_UNINITIALIZED) {
10812                         pr_debug("unregister_netdevice: device %s/%p never was registered\n",
10813                                  dev->name, dev);
10814
10815                         WARN_ON(1);
10816                         list_del(&dev->unreg_list);
10817                         continue;
10818                 }
10819                 dev->dismantle = true;
10820                 BUG_ON(dev->reg_state != NETREG_REGISTERED);
10821         }
10822
10823         /* If device is running, close it first. */
10824         list_for_each_entry(dev, head, unreg_list)
10825                 list_add_tail(&dev->close_list, &close_head);
10826         dev_close_many(&close_head, true);
10827
10828         list_for_each_entry(dev, head, unreg_list) {
10829                 /* And unlink it from device chain. */
10830                 write_lock(&dev_base_lock);
10831                 unlist_netdevice(dev, false);
10832                 dev->reg_state = NETREG_UNREGISTERING;
10833                 write_unlock(&dev_base_lock);
10834         }
10835         flush_all_backlogs();
10836
10837         synchronize_net();
10838
10839         list_for_each_entry(dev, head, unreg_list) {
10840                 struct sk_buff *skb = NULL;
10841
10842                 /* Shutdown queueing discipline. */
10843                 dev_shutdown(dev);
10844
10845                 dev_xdp_uninstall(dev);
10846                 bpf_dev_bound_netdev_unregister(dev);
10847
10848                 netdev_offload_xstats_disable_all(dev);
10849
10850                 /* Notify protocols, that we are about to destroy
10851                  * this device. They should clean all the things.
10852                  */
10853                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10854
10855                 if (!dev->rtnl_link_ops ||
10856                     dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10857                         skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
10858                                                      GFP_KERNEL, NULL, 0,
10859                                                      portid, nlmsg_seq(nlh));
10860
10861                 /*
10862                  *      Flush the unicast and multicast chains
10863                  */
10864                 dev_uc_flush(dev);
10865                 dev_mc_flush(dev);
10866
10867                 netdev_name_node_alt_flush(dev);
10868                 netdev_name_node_free(dev->name_node);
10869
10870                 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
10871
10872                 if (dev->netdev_ops->ndo_uninit)
10873                         dev->netdev_ops->ndo_uninit(dev);
10874
10875                 if (skb)
10876                         rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh);
10877
10878                 /* Notifier chain MUST detach us all upper devices. */
10879                 WARN_ON(netdev_has_any_upper_dev(dev));
10880                 WARN_ON(netdev_has_any_lower_dev(dev));
10881
10882                 /* Remove entries from kobject tree */
10883                 netdev_unregister_kobject(dev);
10884 #ifdef CONFIG_XPS
10885                 /* Remove XPS queueing entries */
10886                 netif_reset_xps_queues_gt(dev, 0);
10887 #endif
10888         }
10889
10890         synchronize_net();
10891
10892         list_for_each_entry(dev, head, unreg_list) {
10893                 netdev_put(dev, &dev->dev_registered_tracker);
10894                 net_set_todo(dev);
10895         }
10896
10897         list_del(head);
10898 }
10899
10900 /**
10901  *      unregister_netdevice_many - unregister many devices
10902  *      @head: list of devices
10903  *
10904  *  Note: As most callers use a stack allocated list_head,
10905  *  we force a list_del() to make sure stack wont be corrupted later.
10906  */
10907 void unregister_netdevice_many(struct list_head *head)
10908 {
10909         unregister_netdevice_many_notify(head, 0, NULL);
10910 }
10911 EXPORT_SYMBOL(unregister_netdevice_many);
10912
10913 /**
10914  *      unregister_netdev - remove device from the kernel
10915  *      @dev: device
10916  *
10917  *      This function shuts down a device interface and removes it
10918  *      from the kernel tables.
10919  *
10920  *      This is just a wrapper for unregister_netdevice that takes
10921  *      the rtnl semaphore.  In general you want to use this and not
10922  *      unregister_netdevice.
10923  */
10924 void unregister_netdev(struct net_device *dev)
10925 {
10926         rtnl_lock();
10927         unregister_netdevice(dev);
10928         rtnl_unlock();
10929 }
10930 EXPORT_SYMBOL(unregister_netdev);
10931
10932 /**
10933  *      __dev_change_net_namespace - move device to different nethost namespace
10934  *      @dev: device
10935  *      @net: network namespace
10936  *      @pat: If not NULL name pattern to try if the current device name
10937  *            is already taken in the destination network namespace.
10938  *      @new_ifindex: If not zero, specifies device index in the target
10939  *                    namespace.
10940  *
10941  *      This function shuts down a device interface and moves it
10942  *      to a new network namespace. On success 0 is returned, on
10943  *      a failure a netagive errno code is returned.
10944  *
10945  *      Callers must hold the rtnl semaphore.
10946  */
10947
10948 int __dev_change_net_namespace(struct net_device *dev, struct net *net,
10949                                const char *pat, int new_ifindex)
10950 {
10951         struct net *net_old = dev_net(dev);
10952         int err, new_nsid;
10953
10954         ASSERT_RTNL();
10955
10956         /* Don't allow namespace local devices to be moved. */
10957         err = -EINVAL;
10958         if (dev->features & NETIF_F_NETNS_LOCAL)
10959                 goto out;
10960
10961         /* Ensure the device has been registrered */
10962         if (dev->reg_state != NETREG_REGISTERED)
10963                 goto out;
10964
10965         /* Get out if there is nothing todo */
10966         err = 0;
10967         if (net_eq(net_old, net))
10968                 goto out;
10969
10970         /* Pick the destination device name, and ensure
10971          * we can use it in the destination network namespace.
10972          */
10973         err = -EEXIST;
10974         if (netdev_name_in_use(net, dev->name)) {
10975                 /* We get here if we can't use the current device name */
10976                 if (!pat)
10977                         goto out;
10978                 err = dev_get_valid_name(net, dev, pat);
10979                 if (err < 0)
10980                         goto out;
10981         }
10982
10983         /* Check that new_ifindex isn't used yet. */
10984         err = -EBUSY;
10985         if (new_ifindex && __dev_get_by_index(net, new_ifindex))
10986                 goto out;
10987
10988         /*
10989          * And now a mini version of register_netdevice unregister_netdevice.
10990          */
10991
10992         /* If device is running close it first. */
10993         dev_close(dev);
10994
10995         /* And unlink it from device chain */
10996         unlist_netdevice(dev, true);
10997
10998         synchronize_net();
10999
11000         /* Shutdown queueing discipline. */
11001         dev_shutdown(dev);
11002
11003         /* Notify protocols, that we are about to destroy
11004          * this device. They should clean all the things.
11005          *
11006          * Note that dev->reg_state stays at NETREG_REGISTERED.
11007          * This is wanted because this way 8021q and macvlan know
11008          * the device is just moving and can keep their slaves up.
11009          */
11010         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11011         rcu_barrier();
11012
11013         new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
11014         /* If there is an ifindex conflict assign a new one */
11015         if (!new_ifindex) {
11016                 if (__dev_get_by_index(net, dev->ifindex))
11017                         new_ifindex = dev_new_index(net);
11018                 else
11019                         new_ifindex = dev->ifindex;
11020         }
11021
11022         rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11023                             new_ifindex);
11024
11025         /*
11026          *      Flush the unicast and multicast chains
11027          */
11028         dev_uc_flush(dev);
11029         dev_mc_flush(dev);
11030
11031         /* Send a netdev-removed uevent to the old namespace */
11032         kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
11033         netdev_adjacent_del_links(dev);
11034
11035         /* Move per-net netdevice notifiers that are following the netdevice */
11036         move_netdevice_notifiers_dev_net(dev, net);
11037
11038         /* Actually switch the network namespace */
11039         dev_net_set(dev, net);
11040         dev->ifindex = new_ifindex;
11041
11042         /* Send a netdev-add uevent to the new namespace */
11043         kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
11044         netdev_adjacent_add_links(dev);
11045
11046         /* Fixup kobjects */
11047         err = device_rename(&dev->dev, dev->name);
11048         WARN_ON(err);
11049
11050         /* Adapt owner in case owning user namespace of target network
11051          * namespace is different from the original one.
11052          */
11053         err = netdev_change_owner(dev, net_old, net);
11054         WARN_ON(err);
11055
11056         /* Add the device back in the hashes */
11057         list_netdevice(dev);
11058
11059         /* Notify protocols, that a new device appeared. */
11060         call_netdevice_notifiers(NETDEV_REGISTER, dev);
11061
11062         /*
11063          *      Prevent userspace races by waiting until the network
11064          *      device is fully setup before sending notifications.
11065          */
11066         rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
11067
11068         synchronize_net();
11069         err = 0;
11070 out:
11071         return err;
11072 }
11073 EXPORT_SYMBOL_GPL(__dev_change_net_namespace);
11074
11075 static int dev_cpu_dead(unsigned int oldcpu)
11076 {
11077         struct sk_buff **list_skb;
11078         struct sk_buff *skb;
11079         unsigned int cpu;
11080         struct softnet_data *sd, *oldsd, *remsd = NULL;
11081
11082         local_irq_disable();
11083         cpu = smp_processor_id();
11084         sd = &per_cpu(softnet_data, cpu);
11085         oldsd = &per_cpu(softnet_data, oldcpu);
11086
11087         /* Find end of our completion_queue. */
11088         list_skb = &sd->completion_queue;
11089         while (*list_skb)
11090                 list_skb = &(*list_skb)->next;
11091         /* Append completion queue from offline CPU. */
11092         *list_skb = oldsd->completion_queue;
11093         oldsd->completion_queue = NULL;
11094
11095         /* Append output queue from offline CPU. */
11096         if (oldsd->output_queue) {
11097                 *sd->output_queue_tailp = oldsd->output_queue;
11098                 sd->output_queue_tailp = oldsd->output_queue_tailp;
11099                 oldsd->output_queue = NULL;
11100                 oldsd->output_queue_tailp = &oldsd->output_queue;
11101         }
11102         /* Append NAPI poll list from offline CPU, with one exception :
11103          * process_backlog() must be called by cpu owning percpu backlog.
11104          * We properly handle process_queue & input_pkt_queue later.
11105          */
11106         while (!list_empty(&oldsd->poll_list)) {
11107                 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11108                                                             struct napi_struct,
11109                                                             poll_list);
11110
11111                 list_del_init(&napi->poll_list);
11112                 if (napi->poll == process_backlog)
11113                         napi->state = 0;
11114                 else
11115                         ____napi_schedule(sd, napi);
11116         }
11117
11118         raise_softirq_irqoff(NET_TX_SOFTIRQ);
11119         local_irq_enable();
11120
11121 #ifdef CONFIG_RPS
11122         remsd = oldsd->rps_ipi_list;
11123         oldsd->rps_ipi_list = NULL;
11124 #endif
11125         /* send out pending IPI's on offline CPU */
11126         net_rps_send_ipi(remsd);
11127
11128         /* Process offline CPU's input_pkt_queue */
11129         while ((skb = __skb_dequeue(&oldsd->process_queue))) {
11130                 netif_rx(skb);
11131                 input_queue_head_incr(oldsd);
11132         }
11133         while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
11134                 netif_rx(skb);
11135                 input_queue_head_incr(oldsd);
11136         }
11137
11138         return 0;
11139 }
11140
11141 /**
11142  *      netdev_increment_features - increment feature set by one
11143  *      @all: current feature set
11144  *      @one: new feature set
11145  *      @mask: mask feature set
11146  *
11147  *      Computes a new feature set after adding a device with feature set
11148  *      @one to the master device with current feature set @all.  Will not
11149  *      enable anything that is off in @mask. Returns the new feature set.
11150  */
11151 netdev_features_t netdev_increment_features(netdev_features_t all,
11152         netdev_features_t one, netdev_features_t mask)
11153 {
11154         if (mask & NETIF_F_HW_CSUM)
11155                 mask |= NETIF_F_CSUM_MASK;
11156         mask |= NETIF_F_VLAN_CHALLENGED;
11157
11158         all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
11159         all &= one | ~NETIF_F_ALL_FOR_ALL;
11160
11161         /* If one device supports hw checksumming, set for all. */
11162         if (all & NETIF_F_HW_CSUM)
11163                 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
11164
11165         return all;
11166 }
11167 EXPORT_SYMBOL(netdev_increment_features);
11168
11169 static struct hlist_head * __net_init netdev_create_hash(void)
11170 {
11171         int i;
11172         struct hlist_head *hash;
11173
11174         hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
11175         if (hash != NULL)
11176                 for (i = 0; i < NETDEV_HASHENTRIES; i++)
11177                         INIT_HLIST_HEAD(&hash[i]);
11178
11179         return hash;
11180 }
11181
11182 /* Initialize per network namespace state */
11183 static int __net_init netdev_init(struct net *net)
11184 {
11185         BUILD_BUG_ON(GRO_HASH_BUCKETS >
11186                      8 * sizeof_field(struct napi_struct, gro_bitmask));
11187
11188         INIT_LIST_HEAD(&net->dev_base_head);
11189
11190         net->dev_name_head = netdev_create_hash();
11191         if (net->dev_name_head == NULL)
11192                 goto err_name;
11193
11194         net->dev_index_head = netdev_create_hash();
11195         if (net->dev_index_head == NULL)
11196                 goto err_idx;
11197
11198         RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
11199
11200         return 0;
11201
11202 err_idx:
11203         kfree(net->dev_name_head);
11204 err_name:
11205         return -ENOMEM;
11206 }
11207
11208 /**
11209  *      netdev_drivername - network driver for the device
11210  *      @dev: network device
11211  *
11212  *      Determine network driver for device.
11213  */
11214 const char *netdev_drivername(const struct net_device *dev)
11215 {
11216         const struct device_driver *driver;
11217         const struct device *parent;
11218         const char *empty = "";
11219
11220         parent = dev->dev.parent;
11221         if (!parent)
11222                 return empty;
11223
11224         driver = parent->driver;
11225         if (driver && driver->name)
11226                 return driver->name;
11227         return empty;
11228 }
11229
11230 static void __netdev_printk(const char *level, const struct net_device *dev,
11231                             struct va_format *vaf)
11232 {
11233         if (dev && dev->dev.parent) {
11234                 dev_printk_emit(level[1] - '0',
11235                                 dev->dev.parent,
11236                                 "%s %s %s%s: %pV",
11237                                 dev_driver_string(dev->dev.parent),
11238                                 dev_name(dev->dev.parent),
11239                                 netdev_name(dev), netdev_reg_state(dev),
11240                                 vaf);
11241         } else if (dev) {
11242                 printk("%s%s%s: %pV",
11243                        level, netdev_name(dev), netdev_reg_state(dev), vaf);
11244         } else {
11245                 printk("%s(NULL net_device): %pV", level, vaf);
11246         }
11247 }
11248
11249 void netdev_printk(const char *level, const struct net_device *dev,
11250                    const char *format, ...)
11251 {
11252         struct va_format vaf;
11253         va_list args;
11254
11255         va_start(args, format);
11256
11257         vaf.fmt = format;
11258         vaf.va = &args;
11259
11260         __netdev_printk(level, dev, &vaf);
11261
11262         va_end(args);
11263 }
11264 EXPORT_SYMBOL(netdev_printk);
11265
11266 #define define_netdev_printk_level(func, level)                 \
11267 void func(const struct net_device *dev, const char *fmt, ...)   \
11268 {                                                               \
11269         struct va_format vaf;                                   \
11270         va_list args;                                           \
11271                                                                 \
11272         va_start(args, fmt);                                    \
11273                                                                 \
11274         vaf.fmt = fmt;                                          \
11275         vaf.va = &args;                                         \
11276                                                                 \
11277         __netdev_printk(level, dev, &vaf);                      \
11278                                                                 \
11279         va_end(args);                                           \
11280 }                                                               \
11281 EXPORT_SYMBOL(func);
11282
11283 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
11284 define_netdev_printk_level(netdev_alert, KERN_ALERT);
11285 define_netdev_printk_level(netdev_crit, KERN_CRIT);
11286 define_netdev_printk_level(netdev_err, KERN_ERR);
11287 define_netdev_printk_level(netdev_warn, KERN_WARNING);
11288 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
11289 define_netdev_printk_level(netdev_info, KERN_INFO);
11290
11291 static void __net_exit netdev_exit(struct net *net)
11292 {
11293         kfree(net->dev_name_head);
11294         kfree(net->dev_index_head);
11295         if (net != &init_net)
11296                 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
11297 }
11298
11299 static struct pernet_operations __net_initdata netdev_net_ops = {
11300         .init = netdev_init,
11301         .exit = netdev_exit,
11302 };
11303
11304 static void __net_exit default_device_exit_net(struct net *net)
11305 {
11306         struct net_device *dev, *aux;
11307         /*
11308          * Push all migratable network devices back to the
11309          * initial network namespace
11310          */
11311         ASSERT_RTNL();
11312         for_each_netdev_safe(net, dev, aux) {
11313                 int err;
11314                 char fb_name[IFNAMSIZ];
11315
11316                 /* Ignore unmoveable devices (i.e. loopback) */
11317                 if (dev->features & NETIF_F_NETNS_LOCAL)
11318                         continue;
11319
11320                 /* Leave virtual devices for the generic cleanup */
11321                 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
11322                         continue;
11323
11324                 /* Push remaining network devices to init_net */
11325                 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
11326                 if (netdev_name_in_use(&init_net, fb_name))
11327                         snprintf(fb_name, IFNAMSIZ, "dev%%d");
11328                 err = dev_change_net_namespace(dev, &init_net, fb_name);
11329                 if (err) {
11330                         pr_emerg("%s: failed to move %s to init_net: %d\n",
11331                                  __func__, dev->name, err);
11332                         BUG();
11333                 }
11334         }
11335 }
11336
11337 static void __net_exit default_device_exit_batch(struct list_head *net_list)
11338 {
11339         /* At exit all network devices most be removed from a network
11340          * namespace.  Do this in the reverse order of registration.
11341          * Do this across as many network namespaces as possible to
11342          * improve batching efficiency.
11343          */
11344         struct net_device *dev;
11345         struct net *net;
11346         LIST_HEAD(dev_kill_list);
11347
11348         rtnl_lock();
11349         list_for_each_entry(net, net_list, exit_list) {
11350                 default_device_exit_net(net);
11351                 cond_resched();
11352         }
11353
11354         list_for_each_entry(net, net_list, exit_list) {
11355                 for_each_netdev_reverse(net, dev) {
11356                         if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
11357                                 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
11358                         else
11359                                 unregister_netdevice_queue(dev, &dev_kill_list);
11360                 }
11361         }
11362         unregister_netdevice_many(&dev_kill_list);
11363         rtnl_unlock();
11364 }
11365
11366 static struct pernet_operations __net_initdata default_device_ops = {
11367         .exit_batch = default_device_exit_batch,
11368 };
11369
11370 /*
11371  *      Initialize the DEV module. At boot time this walks the device list and
11372  *      unhooks any devices that fail to initialise (normally hardware not
11373  *      present) and leaves us with a valid list of present and active devices.
11374  *
11375  */
11376
11377 /*
11378  *       This is called single threaded during boot, so no need
11379  *       to take the rtnl semaphore.
11380  */
11381 static int __init net_dev_init(void)
11382 {
11383         int i, rc = -ENOMEM;
11384
11385         BUG_ON(!dev_boot_phase);
11386
11387         if (dev_proc_init())
11388                 goto out;
11389
11390         if (netdev_kobject_init())
11391                 goto out;
11392
11393         INIT_LIST_HEAD(&ptype_all);
11394         for (i = 0; i < PTYPE_HASH_SIZE; i++)
11395                 INIT_LIST_HEAD(&ptype_base[i]);
11396
11397         if (register_pernet_subsys(&netdev_net_ops))
11398                 goto out;
11399
11400         /*
11401          *      Initialise the packet receive queues.
11402          */
11403
11404         for_each_possible_cpu(i) {
11405                 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
11406                 struct softnet_data *sd = &per_cpu(softnet_data, i);
11407
11408                 INIT_WORK(flush, flush_backlog);
11409
11410                 skb_queue_head_init(&sd->input_pkt_queue);
11411                 skb_queue_head_init(&sd->process_queue);
11412 #ifdef CONFIG_XFRM_OFFLOAD
11413                 skb_queue_head_init(&sd->xfrm_backlog);
11414 #endif
11415                 INIT_LIST_HEAD(&sd->poll_list);
11416                 sd->output_queue_tailp = &sd->output_queue;
11417 #ifdef CONFIG_RPS
11418                 INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
11419                 sd->cpu = i;
11420 #endif
11421                 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
11422                 spin_lock_init(&sd->defer_lock);
11423
11424                 init_gro_hash(&sd->backlog);
11425                 sd->backlog.poll = process_backlog;
11426                 sd->backlog.weight = weight_p;
11427         }
11428
11429         dev_boot_phase = 0;
11430
11431         /* The loopback device is special if any other network devices
11432          * is present in a network namespace the loopback device must
11433          * be present. Since we now dynamically allocate and free the
11434          * loopback device ensure this invariant is maintained by
11435          * keeping the loopback device as the first device on the
11436          * list of network devices.  Ensuring the loopback devices
11437          * is the first device that appears and the last network device
11438          * that disappears.
11439          */
11440         if (register_pernet_device(&loopback_net_ops))
11441                 goto out;
11442
11443         if (register_pernet_device(&default_device_ops))
11444                 goto out;
11445
11446         open_softirq(NET_TX_SOFTIRQ, net_tx_action);
11447         open_softirq(NET_RX_SOFTIRQ, net_rx_action);
11448
11449         rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
11450                                        NULL, dev_cpu_dead);
11451         WARN_ON(rc < 0);
11452         rc = 0;
11453 out:
11454         return rc;
11455 }
11456
11457 subsys_initcall(net_dev_init);