drivers/net/hyperv/netvsc_drv.c

   1 /*
   2  * Copyright (c) 2009, Microsoft Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, see <http://www.gnu.org/licenses/>.
  15  *
  16  * Authors:
  17  *   Haiyang Zhang <haiyangz@microsoft.com>
  18  *   Hank Janssen  <hjanssen@microsoft.com>
  19  */
  20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  21
  22 #include <linux/init.h>
  23 #include <linux/atomic.h>
  24 #include <linux/module.h>
  25 #include <linux/highmem.h>
  26 #include <linux/device.h>
  27 #include <linux/io.h>
  28 #include <linux/delay.h>
  29 #include <linux/netdevice.h>
  30 #include <linux/inetdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/if_vlan.h>
  34 #include <linux/in.h>
  35 #include <linux/slab.h>
  36 #include <linux/rtnetlink.h>
  37 #include <linux/netpoll.h>
  38 #include <linux/reciprocal_div.h>
  39
  40 #include <net/arp.h>
  41 #include <net/route.h>
  42 #include <net/sock.h>
  43 #include <net/pkt_sched.h>
  44 #include <net/checksum.h>
  45 #include <net/ip6_checksum.h>
  46
  47 #include "hyperv_net.h"
  48
  49 #define RING_SIZE_MIN           64
  50 #define NETVSC_MIN_TX_SECTIONS  10
  51 #define NETVSC_DEFAULT_TX       192     /* ~1M */
  52 #define NETVSC_MIN_RX_SECTIONS  10      /* ~64K */
  53 #define NETVSC_DEFAULT_RX       10485   /* Max ~16M */
  54
  55 #define LINKCHANGE_INT (2 * HZ)
  56 #define VF_TAKEOVER_INT (HZ / 10)
  57
  58 static unsigned int ring_size __ro_after_init = 128;
  59 module_param(ring_size, uint, S_IRUGO);
  60 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
  61 unsigned int netvsc_ring_bytes __ro_after_init;
  62 struct reciprocal_value netvsc_ring_reciprocal __ro_after_init;
  63
  64 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
  65                                 NETIF_MSG_LINK | NETIF_MSG_IFUP |
  66                                 NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR |
  67                                 NETIF_MSG_TX_ERR;
  68
  69 static int debug = -1;
  70 module_param(debug, int, S_IRUGO);
  71 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  72
  73 static void netvsc_set_multicast_list(struct net_device *net)
  74 {
  75         struct net_device_context *net_device_ctx = netdev_priv(net);
  76         struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
  77
  78         rndis_filter_update(nvdev);
  79 }
  80
  81 static int netvsc_open(struct net_device *net)
  82 {
  83         struct net_device_context *ndev_ctx = netdev_priv(net);
  84         struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
  85         struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev);
  86         struct rndis_device *rdev;
  87         int ret = 0;
  88
  89         netif_carrier_off(net);
  90
  91         /* Open up the device */
  92         ret = rndis_filter_open(nvdev);
  93         if (ret != 0) {
  94                 netdev_err(net, "unable to open device (ret %d).\n", ret);
  95                 return ret;
  96         }
  97
  98         netif_tx_wake_all_queues(net);
  99
 100         rdev = nvdev->extension;
 101
 102         if (!rdev->link_state)
 103                 netif_carrier_on(net);
 104
 105         if (vf_netdev) {
 106                 /* Setting synthetic device up transparently sets
 107                  * slave as up. If open fails, then slave will be
 108                  * still be offline (and not used).
 109                  */
 110                 ret = dev_open(vf_netdev);
 111                 if (ret)
 112                         netdev_warn(net,
 113                                     "unable to open slave: %s: %d\n",
 114                                     vf_netdev->name, ret);
 115         }
 116         return 0;
 117 }
 118
 119 static int netvsc_close(struct net_device *net)
 120 {
 121         struct net_device_context *net_device_ctx = netdev_priv(net);
 122         struct net_device *vf_netdev
 123                 = rtnl_dereference(net_device_ctx->vf_netdev);
 124         struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 125         int ret = 0;
 126         u32 aread, i, msec = 10, retry = 0, retry_max = 20;
 127         struct vmbus_channel *chn;
 128
 129         netif_tx_disable(net);
 130
 131         /* No need to close rndis filter if it is removed already */
 132         if (!nvdev)
 133                 goto out;
 134
 135         ret = rndis_filter_close(nvdev);
 136         if (ret != 0) {
 137                 netdev_err(net, "unable to close device (ret %d).\n", ret);
 138                 return ret;
 139         }
 140
 141         /* Ensure pending bytes in ring are read */
 142         while (true) {
 143                 aread = 0;
 144                 for (i = 0; i < nvdev->num_chn; i++) {
 145                         chn = nvdev->chan_table[i].channel;
 146                         if (!chn)
 147                                 continue;
 148
 149                         aread = hv_get_bytes_to_read(&chn->inbound);
 150                         if (aread)
 151                                 break;
 152
 153                         aread = hv_get_bytes_to_read(&chn->outbound);
 154                         if (aread)
 155                                 break;
 156                 }
 157
 158                 retry++;
 159                 if (retry > retry_max || aread == 0)
 160                         break;
 161
 162                 msleep(msec);
 163
 164                 if (msec < 1000)
 165                         msec *= 2;
 166         }
 167
 168         if (aread) {
 169                 netdev_err(net, "Ring buffer not empty after closing rndis\n");
 170                 ret = -ETIMEDOUT;
 171         }
 172
 173 out:
 174         if (vf_netdev)
 175                 dev_close(vf_netdev);
 176
 177         return ret;
 178 }
 179
 180 static inline void *init_ppi_data(struct rndis_message *msg,
 181                                   u32 ppi_size, u32 pkt_type)
 182 {
 183         struct rndis_packet *rndis_pkt = &msg->msg.pkt;
 184         struct rndis_per_packet_info *ppi;
 185
 186         rndis_pkt->data_offset += ppi_size;
 187         ppi = (void *)rndis_pkt + rndis_pkt->per_pkt_info_offset
 188                 + rndis_pkt->per_pkt_info_len;
 189
 190         ppi->size = ppi_size;
 191         ppi->type = pkt_type;
 192         ppi->ppi_offset = sizeof(struct rndis_per_packet_info);
 193
 194         rndis_pkt->per_pkt_info_len += ppi_size;
 195
 196         return ppi + 1;
 197 }
 198
 199 /* Azure hosts don't support non-TCP port numbers in hashing for fragmented
 200  * packets. We can use ethtool to change UDP hash level when necessary.
 201  */
 202 static inline u32 netvsc_get_hash(
 203         struct sk_buff *skb,
 204         const struct net_device_context *ndc)
 205 {
 206         struct flow_keys flow;
 207         u32 hash, pkt_proto = 0;
 208         static u32 hashrnd __read_mostly;
 209
 210         net_get_random_once(&hashrnd, sizeof(hashrnd));
 211
 212         if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
 213                 return 0;
 214
 215         switch (flow.basic.ip_proto) {
 216         case IPPROTO_TCP:
 217                 if (flow.basic.n_proto == htons(ETH_P_IP))
 218                         pkt_proto = HV_TCP4_L4HASH;
 219                 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
 220                         pkt_proto = HV_TCP6_L4HASH;
 221
 222                 break;
 223
 224         case IPPROTO_UDP:
 225                 if (flow.basic.n_proto == htons(ETH_P_IP))
 226                         pkt_proto = HV_UDP4_L4HASH;
 227                 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
 228                         pkt_proto = HV_UDP6_L4HASH;
 229
 230                 break;
 231         }
 232
 233         if (pkt_proto & ndc->l4_hash) {
 234                 return skb_get_hash(skb);
 235         } else {
 236                 if (flow.basic.n_proto == htons(ETH_P_IP))
 237                         hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd);
 238                 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
 239                         hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd);
 240                 else
 241                         hash = 0;
 242
 243                 skb_set_hash(skb, hash, PKT_HASH_TYPE_L3);
 244         }
 245
 246         return hash;
 247 }
 248
 249 static inline int netvsc_get_tx_queue(struct net_device *ndev,
 250                                       struct sk_buff *skb, int old_idx)
 251 {
 252         const struct net_device_context *ndc = netdev_priv(ndev);
 253         struct sock *sk = skb->sk;
 254         int q_idx;
 255
 256         q_idx = ndc->tx_table[netvsc_get_hash(skb, ndc) &
 257                               (VRSS_SEND_TAB_SIZE - 1)];
 258
 259         /* If queue index changed record the new value */
 260         if (q_idx != old_idx &&
 261             sk && sk_fullsock(sk) && rcu_access_pointer(sk->sk_dst_cache))
 262                 sk_tx_queue_set(sk, q_idx);
 263
 264         return q_idx;
 265 }
 266
 267 /*
 268  * Select queue for transmit.
 269  *
 270  * If a valid queue has already been assigned, then use that.
 271  * Otherwise compute tx queue based on hash and the send table.
 272  *
 273  * This is basically similar to default (__netdev_pick_tx) with the added step
 274  * of using the host send_table when no other queue has been assigned.
 275  *
 276  * TODO support XPS - but get_xps_queue not exported
 277  */
 278 static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb)
 279 {
 280         int q_idx = sk_tx_queue_get(skb->sk);
 281
 282         if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) {
 283                 /* If forwarding a packet, we use the recorded queue when
 284                  * available for better cache locality.
 285                  */
 286                 if (skb_rx_queue_recorded(skb))
 287                         q_idx = skb_get_rx_queue(skb);
 288                 else
 289                         q_idx = netvsc_get_tx_queue(ndev, skb, q_idx);
 290         }
 291
 292         return q_idx;
 293 }
 294
 295 static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
 296                                void *accel_priv,
 297                                select_queue_fallback_t fallback)
 298 {
 299         struct net_device_context *ndc = netdev_priv(ndev);
 300         struct net_device *vf_netdev;
 301         u16 txq;
 302
 303         rcu_read_lock();
 304         vf_netdev = rcu_dereference(ndc->vf_netdev);
 305         if (vf_netdev) {
 306                 txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 0;
 307                 qdisc_skb_cb(skb)->slave_dev_queue_mapping = skb->queue_mapping;
 308         } else {
 309                 txq = netvsc_pick_tx(ndev, skb);
 310         }
 311         rcu_read_unlock();
 312
 313         while (unlikely(txq >= ndev->real_num_tx_queues))
 314                 txq -= ndev->real_num_tx_queues;
 315
 316         return txq;
 317 }
 318
 319 static u32 fill_pg_buf(struct page *page, u32 offset, u32 len,
 320                        struct hv_page_buffer *pb)
 321 {
 322         int j = 0;
 323
 324         /* Deal with compund pages by ignoring unused part
 325          * of the page.
 326          */
 327         page += (offset >> PAGE_SHIFT);
 328         offset &= ~PAGE_MASK;
 329
 330         while (len > 0) {
 331                 unsigned long bytes;
 332
 333                 bytes = PAGE_SIZE - offset;
 334                 if (bytes > len)
 335                         bytes = len;
 336                 pb[j].pfn = page_to_pfn(page);
 337                 pb[j].offset = offset;
 338                 pb[j].len = bytes;
 339
 340                 offset += bytes;
 341                 len -= bytes;
 342
 343                 if (offset == PAGE_SIZE && len) {
 344                         page++;
 345                         offset = 0;
 346                         j++;
 347                 }
 348         }
 349
 350         return j + 1;
 351 }
 352
 353 static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
 354                            struct hv_netvsc_packet *packet,
 355                            struct hv_page_buffer *pb)
 356 {
 357         u32 slots_used = 0;
 358         char *data = skb->data;
 359         int frags = skb_shinfo(skb)->nr_frags;
 360         int i;
 361
 362         /* The packet is laid out thus:
 363          * 1. hdr: RNDIS header and PPI
 364          * 2. skb linear data
 365          * 3. skb fragment data
 366          */
 367         slots_used += fill_pg_buf(virt_to_page(hdr),
 368                                   offset_in_page(hdr),
 369                                   len, &pb[slots_used]);
 370
 371         packet->rmsg_size = len;
 372         packet->rmsg_pgcnt = slots_used;
 373
 374         slots_used += fill_pg_buf(virt_to_page(data),
 375                                 offset_in_page(data),
 376                                 skb_headlen(skb), &pb[slots_used]);
 377
 378         for (i = 0; i < frags; i++) {
 379                 skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 380
 381                 slots_used += fill_pg_buf(skb_frag_page(frag),
 382                                         frag->page_offset,
 383                                         skb_frag_size(frag), &pb[slots_used]);
 384         }
 385         return slots_used;
 386 }
 387
 388 static int count_skb_frag_slots(struct sk_buff *skb)
 389 {
 390         int i, frags = skb_shinfo(skb)->nr_frags;
 391         int pages = 0;
 392
 393         for (i = 0; i < frags; i++) {
 394                 skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 395                 unsigned long size = skb_frag_size(frag);
 396                 unsigned long offset = frag->page_offset;
 397
 398                 /* Skip unused frames from start of page */
 399                 offset &= ~PAGE_MASK;
 400                 pages += PFN_UP(offset + size);
 401         }
 402         return pages;
 403 }
 404
 405 static int netvsc_get_slots(struct sk_buff *skb)
 406 {
 407         char *data = skb->data;
 408         unsigned int offset = offset_in_page(data);
 409         unsigned int len = skb_headlen(skb);
 410         int slots;
 411         int frag_slots;
 412
 413         slots = DIV_ROUND_UP(offset + len, PAGE_SIZE);
 414         frag_slots = count_skb_frag_slots(skb);
 415         return slots + frag_slots;
 416 }
 417
 418 static u32 net_checksum_info(struct sk_buff *skb)
 419 {
 420         if (skb->protocol == htons(ETH_P_IP)) {
 421                 struct iphdr *ip = ip_hdr(skb);
 422
 423                 if (ip->protocol == IPPROTO_TCP)
 424                         return TRANSPORT_INFO_IPV4_TCP;
 425                 else if (ip->protocol == IPPROTO_UDP)
 426                         return TRANSPORT_INFO_IPV4_UDP;
 427         } else {
 428                 struct ipv6hdr *ip6 = ipv6_hdr(skb);
 429
 430                 if (ip6->nexthdr == IPPROTO_TCP)
 431                         return TRANSPORT_INFO_IPV6_TCP;
 432                 else if (ip6->nexthdr == IPPROTO_UDP)
 433                         return TRANSPORT_INFO_IPV6_UDP;
 434         }
 435
 436         return TRANSPORT_INFO_NOT_IP;
 437 }
 438
 439 /* Send skb on the slave VF device. */
 440 static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev,
 441                           struct sk_buff *skb)
 442 {
 443         struct net_device_context *ndev_ctx = netdev_priv(net);
 444         unsigned int len = skb->len;
 445         int rc;
 446
 447         skb->dev = vf_netdev;
 448         skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;
 449
 450         rc = dev_queue_xmit(skb);
 451         if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) {
 452                 struct netvsc_vf_pcpu_stats *pcpu_stats
 453                         = this_cpu_ptr(ndev_ctx->vf_stats);
 454
 455                 u64_stats_update_begin(&pcpu_stats->syncp);
 456                 pcpu_stats->tx_packets++;
 457                 pcpu_stats->tx_bytes += len;
 458                 u64_stats_update_end(&pcpu_stats->syncp);
 459         } else {
 460                 this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
 461         }
 462
 463         return rc;
 464 }
 465
 466 static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 467 {
 468         struct net_device_context *net_device_ctx = netdev_priv(net);
 469         struct hv_netvsc_packet *packet = NULL;
 470         int ret;
 471         unsigned int num_data_pgs;
 472         struct rndis_message *rndis_msg;
 473         struct net_device *vf_netdev;
 474         u32 rndis_msg_size;
 475         u32 hash;
 476         struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT];
 477
 478         /* if VF is present and up then redirect packets
 479          * already called with rcu_read_lock_bh
 480          */
 481         vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev);
 482         if (vf_netdev && netif_running(vf_netdev) &&
 483             !netpoll_tx_running(net))
 484                 return netvsc_vf_xmit(net, vf_netdev, skb);
 485
 486         /* We will atmost need two pages to describe the rndis
 487          * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
 488          * of pages in a single packet. If skb is scattered around
 489          * more pages we try linearizing it.
 490          */
 491
 492         num_data_pgs = netvsc_get_slots(skb) + 2;
 493
 494         if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
 495                 ++net_device_ctx->eth_stats.tx_scattered;
 496
 497                 if (skb_linearize(skb))
 498                         goto no_memory;
 499
 500                 num_data_pgs = netvsc_get_slots(skb) + 2;
 501                 if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
 502                         ++net_device_ctx->eth_stats.tx_too_big;
 503                         goto drop;
 504                 }
 505         }
 506
 507         /*
 508          * Place the rndis header in the skb head room and
 509          * the skb->cb will be used for hv_netvsc_packet
 510          * structure.
 511          */
 512         ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
 513         if (ret)
 514                 goto no_memory;
 515
 516         /* Use the skb control buffer for building up the packet */
 517         BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
 518                         FIELD_SIZEOF(struct sk_buff, cb));
 519         packet = (struct hv_netvsc_packet *)skb->cb;
 520
 521         packet->q_idx = skb_get_queue_mapping(skb);
 522
 523         packet->total_data_buflen = skb->len;
 524         packet->total_bytes = skb->len;
 525         packet->total_packets = 1;
 526
 527         rndis_msg = (struct rndis_message *)skb->head;
 528
 529         /* Add the rndis header */
 530         rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
 531         rndis_msg->msg_len = packet->total_data_buflen;
 532
 533         rndis_msg->msg.pkt = (struct rndis_packet) {
 534                 .data_offset = sizeof(struct rndis_packet),
 535                 .data_len = packet->total_data_buflen,
 536                 .per_pkt_info_offset = sizeof(struct rndis_packet),
 537         };
 538
 539         rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
 540
 541         hash = skb_get_hash_raw(skb);
 542         if (hash != 0 && net->real_num_tx_queues > 1) {
 543                 u32 *hash_info;
 544
 545                 rndis_msg_size += NDIS_HASH_PPI_SIZE;
 546                 hash_info = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
 547                                           NBL_HASH_VALUE);
 548                 *hash_info = hash;
 549         }
 550
 551         if (skb_vlan_tag_present(skb)) {
 552                 struct ndis_pkt_8021q_info *vlan;
 553
 554                 rndis_msg_size += NDIS_VLAN_PPI_SIZE;
 555                 vlan = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
 556                                      IEEE_8021Q_INFO);
 557
 558                 vlan->value = 0;
 559                 vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
 560                 vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
 561                                 VLAN_PRIO_SHIFT;
 562         }
 563
 564         if (skb_is_gso(skb)) {
 565                 struct ndis_tcp_lso_info *lso_info;
 566
 567                 rndis_msg_size += NDIS_LSO_PPI_SIZE;
 568                 lso_info = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
 569                                          TCP_LARGESEND_PKTINFO);
 570
 571                 lso_info->value = 0;
 572                 lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
 573                 if (skb->protocol == htons(ETH_P_IP)) {
 574                         lso_info->lso_v2_transmit.ip_version =
 575                                 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
 576                         ip_hdr(skb)->tot_len = 0;
 577                         ip_hdr(skb)->check = 0;
 578                         tcp_hdr(skb)->check =
 579                                 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
 580                                                    ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
 581                 } else {
 582                         lso_info->lso_v2_transmit.ip_version =
 583                                 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
 584                         ipv6_hdr(skb)->payload_len = 0;
 585                         tcp_hdr(skb)->check =
 586                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
 587                                                  &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
 588                 }
 589                 lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb);
 590                 lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
 591         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 592                 if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) {
 593                         struct ndis_tcp_ip_checksum_info *csum_info;
 594
 595                         rndis_msg_size += NDIS_CSUM_PPI_SIZE;
 596                         csum_info = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
 597                                                   TCPIP_CHKSUM_PKTINFO);
 598
 599                         csum_info->value = 0;
 600                         csum_info->transmit.tcp_header_offset = skb_transport_offset(skb);
 601
 602                         if (skb->protocol == htons(ETH_P_IP)) {
 603                                 csum_info->transmit.is_ipv4 = 1;
 604
 605                                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
 606                                         csum_info->transmit.tcp_checksum = 1;
 607                                 else
 608                                         csum_info->transmit.udp_checksum = 1;
 609                         } else {
 610                                 csum_info->transmit.is_ipv6 = 1;
 611
 612                                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
 613                                         csum_info->transmit.tcp_checksum = 1;
 614                                 else
 615                                         csum_info->transmit.udp_checksum = 1;
 616                         }
 617                 } else {
 618                         /* Can't do offload of this type of checksum */
 619                         if (skb_checksum_help(skb))
 620                                 goto drop;
 621                 }
 622         }
 623
 624         /* Start filling in the page buffers with the rndis hdr */
 625         rndis_msg->msg_len += rndis_msg_size;
 626         packet->total_data_buflen = rndis_msg->msg_len;
 627         packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
 628                                                skb, packet, pb);
 629
 630         /* timestamp packet in software */
 631         skb_tx_timestamp(skb);
 632
 633         ret = netvsc_send(net_device_ctx, packet, rndis_msg, pb, skb);
 634         if (likely(ret == 0))
 635                 return NETDEV_TX_OK;
 636
 637         if (ret == -EAGAIN) {
 638                 ++net_device_ctx->eth_stats.tx_busy;
 639                 return NETDEV_TX_BUSY;
 640         }
 641
 642         if (ret == -ENOSPC)
 643                 ++net_device_ctx->eth_stats.tx_no_space;
 644
 645 drop:
 646         dev_kfree_skb_any(skb);
 647         net->stats.tx_dropped++;
 648
 649         return NETDEV_TX_OK;
 650
 651 no_memory:
 652         ++net_device_ctx->eth_stats.tx_no_memory;
 653         goto drop;
 654 }
 655
 656 /*
 657  * netvsc_linkstatus_callback - Link up/down notification
 658  */
 659 void netvsc_linkstatus_callback(struct hv_device *device_obj,
 660                                 struct rndis_message *resp)
 661 {
 662         struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
 663         struct net_device *net;
 664         struct net_device_context *ndev_ctx;
 665         struct netvsc_reconfig *event;
 666         unsigned long flags;
 667
 668         net = hv_get_drvdata(device_obj);
 669
 670         if (!net)
 671                 return;
 672
 673         ndev_ctx = netdev_priv(net);
 674
 675         /* Update the physical link speed when changing to another vSwitch */
 676         if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
 677                 u32 speed;
 678
 679                 speed = *(u32 *)((void *)indicate
 680                                  + indicate->status_buf_offset) / 10000;
 681                 ndev_ctx->speed = speed;
 682                 return;
 683         }
 684
 685         /* Handle these link change statuses below */
 686         if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE &&
 687             indicate->status != RNDIS_STATUS_MEDIA_CONNECT &&
 688             indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT)
 689                 return;
 690
 691         if (net->reg_state != NETREG_REGISTERED)
 692                 return;
 693
 694         event = kzalloc(sizeof(*event), GFP_ATOMIC);
 695         if (!event)
 696                 return;
 697         event->event = indicate->status;
 698
 699         spin_lock_irqsave(&ndev_ctx->lock, flags);
 700         list_add_tail(&event->list, &ndev_ctx->reconfig_events);
 701         spin_unlock_irqrestore(&ndev_ctx->lock, flags);
 702
 703         schedule_delayed_work(&ndev_ctx->dwork, 0);
 704 }
 705
 706 static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
 707                                              struct napi_struct *napi,
 708                                              const struct ndis_tcp_ip_checksum_info *csum_info,
 709                                              const struct ndis_pkt_8021q_info *vlan,
 710                                              void *data, u32 buflen)
 711 {
 712         struct sk_buff *skb;
 713
 714         skb = napi_alloc_skb(napi, buflen);
 715         if (!skb)
 716                 return skb;
 717
 718         /*
 719          * Copy to skb. This copy is needed here since the memory pointed by
 720          * hv_netvsc_packet cannot be deallocated
 721          */
 722         skb_put_data(skb, data, buflen);
 723
 724         skb->protocol = eth_type_trans(skb, net);
 725
 726         /* skb is already created with CHECKSUM_NONE */
 727         skb_checksum_none_assert(skb);
 728
 729         /*
 730          * In Linux, the IP checksum is always checked.
 731          * Do L4 checksum offload if enabled and present.
 732          */
 733         if (csum_info && (net->features & NETIF_F_RXCSUM)) {
 734                 if (csum_info->receive.tcp_checksum_succeeded ||
 735                     csum_info->receive.udp_checksum_succeeded)
 736                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 737         }
 738
 739         if (vlan) {
 740                 u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT);
 741
 742                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 743                                        vlan_tci);
 744         }
 745
 746         return skb;
 747 }
 748
 749 /*
 750  * netvsc_recv_callback -  Callback when we receive a packet from the
 751  * "wire" on the specified device.
 752  */
 753 int netvsc_recv_callback(struct net_device *net,
 754                          struct vmbus_channel *channel,
 755                          void  *data, u32 len,
 756                          const struct ndis_tcp_ip_checksum_info *csum_info,
 757                          const struct ndis_pkt_8021q_info *vlan)
 758 {
 759         struct net_device_context *net_device_ctx = netdev_priv(net);
 760         struct netvsc_device *net_device;
 761         u16 q_idx = channel->offermsg.offer.sub_channel_index;
 762         struct netvsc_channel *nvchan;
 763         struct sk_buff *skb;
 764         struct netvsc_stats *rx_stats;
 765
 766         if (net->reg_state != NETREG_REGISTERED)
 767                 return NVSP_STAT_FAIL;
 768
 769         rcu_read_lock();
 770         net_device = rcu_dereference(net_device_ctx->nvdev);
 771         if (unlikely(!net_device))
 772                 goto drop;
 773
 774         nvchan = &net_device->chan_table[q_idx];
 775
 776         /* Allocate a skb - TODO direct I/O to pages? */
 777         skb = netvsc_alloc_recv_skb(net, &nvchan->napi,
 778                                     csum_info, vlan, data, len);
 779         if (unlikely(!skb)) {
 780 drop:
 781                 ++net->stats.rx_dropped;
 782                 rcu_read_unlock();
 783                 return NVSP_STAT_FAIL;
 784         }
 785
 786         skb_record_rx_queue(skb, q_idx);
 787
 788         /*
 789          * Even if injecting the packet, record the statistics
 790          * on the synthetic device because modifying the VF device
 791          * statistics will not work correctly.
 792          */
 793         rx_stats = &nvchan->rx_stats;
 794         u64_stats_update_begin(&rx_stats->syncp);
 795         rx_stats->packets++;
 796         rx_stats->bytes += len;
 797
 798         if (skb->pkt_type == PACKET_BROADCAST)
 799                 ++rx_stats->broadcast;
 800         else if (skb->pkt_type == PACKET_MULTICAST)
 801                 ++rx_stats->multicast;
 802         u64_stats_update_end(&rx_stats->syncp);
 803
 804         napi_gro_receive(&nvchan->napi, skb);
 805         rcu_read_unlock();
 806
 807         return 0;
 808 }
 809
 810 static void netvsc_get_drvinfo(struct net_device *net,
 811                                struct ethtool_drvinfo *info)
 812 {
 813         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
 814         strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
 815 }
 816
 817 static void netvsc_get_channels(struct net_device *net,
 818                                 struct ethtool_channels *channel)
 819 {
 820         struct net_device_context *net_device_ctx = netdev_priv(net);
 821         struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 822
 823         if (nvdev) {
 824                 channel->max_combined   = nvdev->max_chn;
 825                 channel->combined_count = nvdev->num_chn;
 826         }
 827 }
 828
 829 static int netvsc_set_channels(struct net_device *net,
 830                                struct ethtool_channels *channels)
 831 {
 832         struct net_device_context *net_device_ctx = netdev_priv(net);
 833         struct hv_device *dev = net_device_ctx->device_ctx;
 834         struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 835         unsigned int orig, count = channels->combined_count;
 836         struct netvsc_device_info device_info;
 837         bool was_opened;
 838         int ret = 0;
 839
 840         /* We do not support separate count for rx, tx, or other */
 841         if (count == 0 ||
 842             channels->rx_count || channels->tx_count || channels->other_count)
 843                 return -EINVAL;
 844
 845         if (!nvdev || nvdev->destroy)
 846                 return -ENODEV;
 847
 848         if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5)
 849                 return -EINVAL;
 850
 851         if (count > nvdev->max_chn)
 852                 return -EINVAL;
 853
 854         orig = nvdev->num_chn;
 855         was_opened = rndis_filter_opened(nvdev);
 856         if (was_opened)
 857                 rndis_filter_close(nvdev);
 858
 859         memset(&device_info, 0, sizeof(device_info));
 860         device_info.num_chn = count;
 861         device_info.send_sections = nvdev->send_section_cnt;
 862         device_info.send_section_size = nvdev->send_section_size;
 863         device_info.recv_sections = nvdev->recv_section_cnt;
 864         device_info.recv_section_size = nvdev->recv_section_size;
 865
 866         rndis_filter_device_remove(dev, nvdev);
 867
 868         nvdev = rndis_filter_device_add(dev, &device_info);
 869         if (IS_ERR(nvdev)) {
 870                 ret = PTR_ERR(nvdev);
 871                 device_info.num_chn = orig;
 872                 nvdev = rndis_filter_device_add(dev, &device_info);
 873
 874                 if (IS_ERR(nvdev)) {
 875                         netdev_err(net, "restoring channel setting failed: %ld\n",
 876                                    PTR_ERR(nvdev));
 877                         return ret;
 878                 }
 879         }
 880
 881         if (was_opened)
 882                 rndis_filter_open(nvdev);
 883
 884         /* We may have missed link change notifications */
 885         net_device_ctx->last_reconfig = 0;
 886         schedule_delayed_work(&net_device_ctx->dwork, 0);
 887
 888         return ret;
 889 }
 890
 891 static bool
 892 netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
 893 {
 894         struct ethtool_link_ksettings diff1 = *cmd;
 895         struct ethtool_link_ksettings diff2 = {};
 896
 897         diff1.base.speed = 0;
 898         diff1.base.duplex = 0;
 899         /* advertising and cmd are usually set */
 900         ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
 901         diff1.base.cmd = 0;
 902         /* We set port to PORT_OTHER */
 903         diff2.base.port = PORT_OTHER;
 904
 905         return !memcmp(&diff1, &diff2, sizeof(diff1));
 906 }
 907
 908 static void netvsc_init_settings(struct net_device *dev)
 909 {
 910         struct net_device_context *ndc = netdev_priv(dev);
 911
 912         ndc->l4_hash = HV_DEFAULT_L4HASH;
 913
 914         ndc->speed = SPEED_UNKNOWN;
 915         ndc->duplex = DUPLEX_FULL;
 916 }
 917
 918 static int netvsc_get_link_ksettings(struct net_device *dev,
 919                                      struct ethtool_link_ksettings *cmd)
 920 {
 921         struct net_device_context *ndc = netdev_priv(dev);
 922
 923         cmd->base.speed = ndc->speed;
 924         cmd->base.duplex = ndc->duplex;
 925         cmd->base.port = PORT_OTHER;
 926
 927         return 0;
 928 }
 929
 930 static int netvsc_set_link_ksettings(struct net_device *dev,
 931                                      const struct ethtool_link_ksettings *cmd)
 932 {
 933         struct net_device_context *ndc = netdev_priv(dev);
 934         u32 speed;
 935
 936         speed = cmd->base.speed;
 937         if (!ethtool_validate_speed(speed) ||
 938             !ethtool_validate_duplex(cmd->base.duplex) ||
 939             !netvsc_validate_ethtool_ss_cmd(cmd))
 940                 return -EINVAL;
 941
 942         ndc->speed = speed;
 943         ndc->duplex = cmd->base.duplex;
 944
 945         return 0;
 946 }
 947
 948 static int netvsc_change_mtu(struct net_device *ndev, int mtu)
 949 {
 950         struct net_device_context *ndevctx = netdev_priv(ndev);
 951         struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
 952         struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
 953         struct hv_device *hdev = ndevctx->device_ctx;
 954         int orig_mtu = ndev->mtu;
 955         struct netvsc_device_info device_info;
 956         bool was_opened;
 957         int ret = 0;
 958
 959         if (!nvdev || nvdev->destroy)
 960                 return -ENODEV;
 961
 962         /* Change MTU of underlying VF netdev first. */
 963         if (vf_netdev) {
 964                 ret = dev_set_mtu(vf_netdev, mtu);
 965                 if (ret)
 966                         return ret;
 967         }
 968
 969         netif_device_detach(ndev);
 970         was_opened = rndis_filter_opened(nvdev);
 971         if (was_opened)
 972                 rndis_filter_close(nvdev);
 973
 974         memset(&device_info, 0, sizeof(device_info));
 975         device_info.num_chn = nvdev->num_chn;
 976         device_info.send_sections = nvdev->send_section_cnt;
 977         device_info.send_section_size = nvdev->send_section_size;
 978         device_info.recv_sections = nvdev->recv_section_cnt;
 979         device_info.recv_section_size = nvdev->recv_section_size;
 980
 981         rndis_filter_device_remove(hdev, nvdev);
 982
 983         ndev->mtu = mtu;
 984
 985         nvdev = rndis_filter_device_add(hdev, &device_info);
 986         if (IS_ERR(nvdev)) {
 987                 ret = PTR_ERR(nvdev);
 988
 989                 /* Attempt rollback to original MTU */
 990                 ndev->mtu = orig_mtu;
 991                 nvdev = rndis_filter_device_add(hdev, &device_info);
 992
 993                 if (vf_netdev)
 994                         dev_set_mtu(vf_netdev, orig_mtu);
 995
 996                 if (IS_ERR(nvdev)) {
 997                         netdev_err(ndev, "restoring mtu failed: %ld\n",
 998                                    PTR_ERR(nvdev));
 999                         return ret;
1000                 }
1001         }
1002
1003         if (was_opened)
1004                 rndis_filter_open(nvdev);
1005
1006         netif_device_attach(ndev);
1007
1008         /* We may have missed link change notifications */
1009         schedule_delayed_work(&ndevctx->dwork, 0);
1010
1011         return ret;
1012 }
1013
1014 static void netvsc_get_vf_stats(struct net_device *net,
1015                                 struct netvsc_vf_pcpu_stats *tot)
1016 {
1017         struct net_device_context *ndev_ctx = netdev_priv(net);
1018         int i;
1019
1020         memset(tot, 0, sizeof(*tot));
1021
1022         for_each_possible_cpu(i) {
1023                 const struct netvsc_vf_pcpu_stats *stats
1024                         = per_cpu_ptr(ndev_ctx->vf_stats, i);
1025                 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
1026                 unsigned int start;
1027
1028                 do {
1029                         start = u64_stats_fetch_begin_irq(&stats->syncp);
1030                         rx_packets = stats->rx_packets;
1031                         tx_packets = stats->tx_packets;
1032                         rx_bytes = stats->rx_bytes;
1033                         tx_bytes = stats->tx_bytes;
1034                 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1035
1036                 tot->rx_packets += rx_packets;
1037                 tot->tx_packets += tx_packets;
1038                 tot->rx_bytes   += rx_bytes;
1039                 tot->tx_bytes   += tx_bytes;
1040                 tot->tx_dropped += stats->tx_dropped;
1041         }
1042 }
1043
1044 static void netvsc_get_stats64(struct net_device *net,
1045                                struct rtnl_link_stats64 *t)
1046 {
1047         struct net_device_context *ndev_ctx = netdev_priv(net);
1048         struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev);
1049         struct netvsc_vf_pcpu_stats vf_tot;
1050         int i;
1051
1052         if (!nvdev)
1053                 return;
1054
1055         netdev_stats_to_stats64(t, &net->stats);
1056
1057         netvsc_get_vf_stats(net, &vf_tot);
1058         t->rx_packets += vf_tot.rx_packets;
1059         t->tx_packets += vf_tot.tx_packets;
1060         t->rx_bytes   += vf_tot.rx_bytes;
1061         t->tx_bytes   += vf_tot.tx_bytes;
1062         t->tx_dropped += vf_tot.tx_dropped;
1063
1064         for (i = 0; i < nvdev->num_chn; i++) {
1065                 const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
1066                 const struct netvsc_stats *stats;
1067                 u64 packets, bytes, multicast;
1068                 unsigned int start;
1069
1070                 stats = &nvchan->tx_stats;
1071                 do {
1072                         start = u64_stats_fetch_begin_irq(&stats->syncp);
1073                         packets = stats->packets;
1074                         bytes = stats->bytes;
1075                 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1076
1077                 t->tx_bytes     += bytes;
1078                 t->tx_packets   += packets;
1079
1080                 stats = &nvchan->rx_stats;
1081                 do {
1082                         start = u64_stats_fetch_begin_irq(&stats->syncp);
1083                         packets = stats->packets;
1084                         bytes = stats->bytes;
1085                         multicast = stats->multicast + stats->broadcast;
1086                 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1087
1088                 t->rx_bytes     += bytes;
1089                 t->rx_packets   += packets;
1090                 t->multicast    += multicast;
1091         }
1092 }
1093
1094 static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
1095 {
1096         struct net_device_context *ndc = netdev_priv(ndev);
1097         struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev);
1098         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1099         struct sockaddr *addr = p;
1100         int err;
1101
1102         err = eth_prepare_mac_addr_change(ndev, p);
1103         if (err)
1104                 return err;
1105
1106         if (!nvdev)
1107                 return -ENODEV;
1108
1109         if (vf_netdev) {
1110                 err = dev_set_mac_address(vf_netdev, addr);
1111                 if (err)
1112                         return err;
1113         }
1114
1115         err = rndis_filter_set_device_mac(nvdev, addr->sa_data);
1116         if (!err) {
1117                 eth_commit_mac_addr_change(ndev, p);
1118         } else if (vf_netdev) {
1119                 /* rollback change on VF */
1120                 memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN);
1121                 dev_set_mac_address(vf_netdev, addr);
1122         }
1123
1124         return err;
1125 }
1126
1127 static const struct {
1128         char name[ETH_GSTRING_LEN];
1129         u16 offset;
1130 } netvsc_stats[] = {
1131         { "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) },
1132         { "tx_no_memory",  offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
1133         { "tx_no_space",  offsetof(struct netvsc_ethtool_stats, tx_no_space) },
1134         { "tx_too_big",   offsetof(struct netvsc_ethtool_stats, tx_too_big) },
1135         { "tx_busy",      offsetof(struct netvsc_ethtool_stats, tx_busy) },
1136         { "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) },
1137         { "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) },
1138         { "stop_queue", offsetof(struct netvsc_ethtool_stats, stop_queue) },
1139         { "wake_queue", offsetof(struct netvsc_ethtool_stats, wake_queue) },
1140 }, vf_stats[] = {
1141         { "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) },
1142         { "vf_rx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) },
1143         { "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) },
1144         { "vf_tx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) },
1145         { "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) },
1146 };
1147
1148 #define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats)
1149 #define NETVSC_VF_STATS_LEN     ARRAY_SIZE(vf_stats)
1150
1151 /* 4 statistics per queue (rx/tx packets/bytes) */
1152 #define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
1153
1154 static int netvsc_get_sset_count(struct net_device *dev, int string_set)
1155 {
1156         struct net_device_context *ndc = netdev_priv(dev);
1157         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1158
1159         if (!nvdev)
1160                 return -ENODEV;
1161
1162         switch (string_set) {
1163         case ETH_SS_STATS:
1164                 return NETVSC_GLOBAL_STATS_LEN
1165                         + NETVSC_VF_STATS_LEN
1166                         + NETVSC_QUEUE_STATS_LEN(nvdev);
1167         default:
1168                 return -EINVAL;
1169         }
1170 }
1171
1172 static void netvsc_get_ethtool_stats(struct net_device *dev,
1173                                      struct ethtool_stats *stats, u64 *data)
1174 {
1175         struct net_device_context *ndc = netdev_priv(dev);
1176         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1177         const void *nds = &ndc->eth_stats;
1178         const struct netvsc_stats *qstats;
1179         struct netvsc_vf_pcpu_stats sum;
1180         unsigned int start;
1181         u64 packets, bytes;
1182         int i, j;
1183
1184         if (!nvdev)
1185                 return;
1186
1187         for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
1188                 data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
1189
1190         netvsc_get_vf_stats(dev, &sum);
1191         for (j = 0; j < NETVSC_VF_STATS_LEN; j++)
1192                 data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset);
1193
1194         for (j = 0; j < nvdev->num_chn; j++) {
1195                 qstats = &nvdev->chan_table[j].tx_stats;
1196
1197                 do {
1198                         start = u64_stats_fetch_begin_irq(&qstats->syncp);
1199                         packets = qstats->packets;
1200                         bytes = qstats->bytes;
1201                 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1202                 data[i++] = packets;
1203                 data[i++] = bytes;
1204
1205                 qstats = &nvdev->chan_table[j].rx_stats;
1206                 do {
1207                         start = u64_stats_fetch_begin_irq(&qstats->syncp);
1208                         packets = qstats->packets;
1209                         bytes = qstats->bytes;
1210                 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1211                 data[i++] = packets;
1212                 data[i++] = bytes;
1213         }
1214 }
1215
1216 static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1217 {
1218         struct net_device_context *ndc = netdev_priv(dev);
1219         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1220         u8 *p = data;
1221         int i;
1222
1223         if (!nvdev)
1224                 return;
1225
1226         switch (stringset) {
1227         case ETH_SS_STATS:
1228                 for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) {
1229                         memcpy(p, netvsc_stats[i].name, ETH_GSTRING_LEN);
1230                         p += ETH_GSTRING_LEN;
1231                 }
1232
1233                 for (i = 0; i < ARRAY_SIZE(vf_stats); i++) {
1234                         memcpy(p, vf_stats[i].name, ETH_GSTRING_LEN);
1235                         p += ETH_GSTRING_LEN;
1236                 }
1237
1238                 for (i = 0; i < nvdev->num_chn; i++) {
1239                         sprintf(p, "tx_queue_%u_packets", i);
1240                         p += ETH_GSTRING_LEN;
1241                         sprintf(p, "tx_queue_%u_bytes", i);
1242                         p += ETH_GSTRING_LEN;
1243                         sprintf(p, "rx_queue_%u_packets", i);
1244                         p += ETH_GSTRING_LEN;
1245                         sprintf(p, "rx_queue_%u_bytes", i);
1246                         p += ETH_GSTRING_LEN;
1247                 }
1248
1249                 break;
1250         }
1251 }
1252
1253 static int
1254 netvsc_get_rss_hash_opts(struct net_device_context *ndc,
1255                          struct ethtool_rxnfc *info)
1256 {
1257         const u32 l4_flag = RXH_L4_B_0_1 | RXH_L4_B_2_3;
1258
1259         info->data = RXH_IP_SRC | RXH_IP_DST;
1260
1261         switch (info->flow_type) {
1262         case TCP_V4_FLOW:
1263                 if (ndc->l4_hash & HV_TCP4_L4HASH)
1264                         info->data |= l4_flag;
1265
1266                 break;
1267
1268         case TCP_V6_FLOW:
1269                 if (ndc->l4_hash & HV_TCP6_L4HASH)
1270                         info->data |= l4_flag;
1271
1272                 break;
1273
1274         case UDP_V4_FLOW:
1275                 if (ndc->l4_hash & HV_UDP4_L4HASH)
1276                         info->data |= l4_flag;
1277
1278                 break;
1279
1280         case UDP_V6_FLOW:
1281                 if (ndc->l4_hash & HV_UDP6_L4HASH)
1282                         info->data |= l4_flag;
1283
1284                 break;
1285
1286         case IPV4_FLOW:
1287         case IPV6_FLOW:
1288                 break;
1289         default:
1290                 info->data = 0;
1291                 break;
1292         }
1293
1294         return 0;
1295 }
1296
1297 static int
1298 netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1299                  u32 *rules)
1300 {
1301         struct net_device_context *ndc = netdev_priv(dev);
1302         struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1303
1304         if (!nvdev)
1305                 return -ENODEV;
1306
1307         switch (info->cmd) {
1308         case ETHTOOL_GRXRINGS:
1309                 info->data = nvdev->num_chn;
1310                 return 0;
1311
1312         case ETHTOOL_GRXFH:
1313                 return netvsc_get_rss_hash_opts(ndc, info);
1314         }
1315         return -EOPNOTSUPP;
1316 }
1317
1318 static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
1319                                     struct ethtool_rxnfc *info)
1320 {
1321         if (info->data == (RXH_IP_SRC | RXH_IP_DST |
1322                            RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
1323                 switch (info->flow_type) {
1324                 case TCP_V4_FLOW:
1325                         ndc->l4_hash |= HV_TCP4_L4HASH;
1326                         break;
1327
1328                 case TCP_V6_FLOW:
1329                         ndc->l4_hash |= HV_TCP6_L4HASH;
1330                         break;
1331
1332                 case UDP_V4_FLOW:
1333                         ndc->l4_hash |= HV_UDP4_L4HASH;
1334                         break;
1335
1336                 case UDP_V6_FLOW:
1337                         ndc->l4_hash |= HV_UDP6_L4HASH;
1338                         break;
1339
1340                 default:
1341                         return -EOPNOTSUPP;
1342                 }
1343
1344                 return 0;
1345         }
1346
1347         if (info->data == (RXH_IP_SRC | RXH_IP_DST)) {
1348                 switch (info->flow_type) {
1349                 case TCP_V4_FLOW:
1350                         ndc->l4_hash &= ~HV_TCP4_L4HASH;
1351                         break;
1352
1353                 case TCP_V6_FLOW:
1354                         ndc->l4_hash &= ~HV_TCP6_L4HASH;
1355                         break;
1356
1357                 case UDP_V4_FLOW:
1358                         ndc->l4_hash &= ~HV_UDP4_L4HASH;
1359                         break;
1360
1361                 case UDP_V6_FLOW:
1362                         ndc->l4_hash &= ~HV_UDP6_L4HASH;
1363                         break;
1364
1365                 default:
1366                         return -EOPNOTSUPP;
1367                 }
1368
1369                 return 0;
1370         }
1371
1372         return -EOPNOTSUPP;
1373 }
1374
1375 static int
1376 netvsc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info)
1377 {
1378         struct net_device_context *ndc = netdev_priv(ndev);
1379
1380         if (info->cmd == ETHTOOL_SRXFH)
1381                 return netvsc_set_rss_hash_opts(ndc, info);
1382
1383         return -EOPNOTSUPP;
1384 }
1385
1386 #ifdef CONFIG_NET_POLL_CONTROLLER
1387 static void netvsc_poll_controller(struct net_device *dev)
1388 {
1389         struct net_device_context *ndc = netdev_priv(dev);
1390         struct netvsc_device *ndev;
1391         int i;
1392
1393         rcu_read_lock();
1394         ndev = rcu_dereference(ndc->nvdev);
1395         if (ndev) {
1396                 for (i = 0; i < ndev->num_chn; i++) {
1397                         struct netvsc_channel *nvchan = &ndev->chan_table[i];
1398
1399                         napi_schedule(&nvchan->napi);
1400                 }
1401         }
1402         rcu_read_unlock();
1403 }
1404 #endif
1405
1406 static u32 netvsc_get_rxfh_key_size(struct net_device *dev)
1407 {
1408         return NETVSC_HASH_KEYLEN;
1409 }
1410
1411 static u32 netvsc_rss_indir_size(struct net_device *dev)
1412 {
1413         return ITAB_NUM;
1414 }
1415
1416 static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
1417                            u8 *hfunc)
1418 {
1419         struct net_device_context *ndc = netdev_priv(dev);
1420         struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1421         struct rndis_device *rndis_dev;
1422         int i;
1423
1424         if (!ndev)
1425                 return -ENODEV;
1426
1427         if (hfunc)
1428                 *hfunc = ETH_RSS_HASH_TOP;      /* Toeplitz */
1429
1430         rndis_dev = ndev->extension;
1431         if (indir) {
1432                 for (i = 0; i < ITAB_NUM; i++)
1433                         indir[i] = rndis_dev->rx_table[i];
1434         }
1435
1436         if (key)
1437                 memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN);
1438
1439         return 0;
1440 }
1441
1442 static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir,
1443                            const u8 *key, const u8 hfunc)
1444 {
1445         struct net_device_context *ndc = netdev_priv(dev);
1446         struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1447         struct rndis_device *rndis_dev;
1448         int i;
1449
1450         if (!ndev)
1451                 return -ENODEV;
1452
1453         if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
1454                 return -EOPNOTSUPP;
1455
1456         rndis_dev = ndev->extension;
1457         if (indir) {
1458                 for (i = 0; i < ITAB_NUM; i++)
1459                         if (indir[i] >= ndev->num_chn)
1460                                 return -EINVAL;
1461
1462                 for (i = 0; i < ITAB_NUM; i++)
1463                         rndis_dev->rx_table[i] = indir[i];
1464         }
1465
1466         if (!key) {
1467                 if (!indir)
1468                         return 0;
1469
1470                 key = rndis_dev->rss_key;
1471         }
1472
1473         return rndis_filter_set_rss_param(rndis_dev, key);
1474 }
1475
1476 /* Hyper-V RNDIS protocol does not have ring in the HW sense.
1477  * It does have pre-allocated receive area which is divided into sections.
1478  */
1479 static void __netvsc_get_ringparam(struct netvsc_device *nvdev,
1480                                    struct ethtool_ringparam *ring)
1481 {
1482         u32 max_buf_size;
1483
1484         ring->rx_pending = nvdev->recv_section_cnt;
1485         ring->tx_pending = nvdev->send_section_cnt;
1486
1487         if (nvdev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
1488                 max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
1489         else
1490                 max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
1491
1492         ring->rx_max_pending = max_buf_size / nvdev->recv_section_size;
1493         ring->tx_max_pending = NETVSC_SEND_BUFFER_SIZE
1494                 / nvdev->send_section_size;
1495 }
1496
1497 static void netvsc_get_ringparam(struct net_device *ndev,
1498                                  struct ethtool_ringparam *ring)
1499 {
1500         struct net_device_context *ndevctx = netdev_priv(ndev);
1501         struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1502
1503         if (!nvdev)
1504                 return;
1505
1506         __netvsc_get_ringparam(nvdev, ring);
1507 }
1508
1509 static int netvsc_set_ringparam(struct net_device *ndev,
1510                                 struct ethtool_ringparam *ring)
1511 {
1512         struct net_device_context *ndevctx = netdev_priv(ndev);
1513         struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1514         struct hv_device *hdev = ndevctx->device_ctx;
1515         struct netvsc_device_info device_info;
1516         struct ethtool_ringparam orig;
1517         u32 new_tx, new_rx;
1518         bool was_opened;
1519         int ret = 0;
1520
1521         if (!nvdev || nvdev->destroy)
1522                 return -ENODEV;
1523
1524         memset(&orig, 0, sizeof(orig));
1525         __netvsc_get_ringparam(nvdev, &orig);
1526
1527         new_tx = clamp_t(u32, ring->tx_pending,
1528                          NETVSC_MIN_TX_SECTIONS, orig.tx_max_pending);
1529         new_rx = clamp_t(u32, ring->rx_pending,
1530                          NETVSC_MIN_RX_SECTIONS, orig.rx_max_pending);
1531
1532         if (new_tx == orig.tx_pending &&
1533             new_rx == orig.rx_pending)
1534                 return 0;        /* no change */
1535
1536         memset(&device_info, 0, sizeof(device_info));
1537         device_info.num_chn = nvdev->num_chn;
1538         device_info.send_sections = new_tx;
1539         device_info.send_section_size = nvdev->send_section_size;
1540         device_info.recv_sections = new_rx;
1541         device_info.recv_section_size = nvdev->recv_section_size;
1542
1543         netif_device_detach(ndev);
1544         was_opened = rndis_filter_opened(nvdev);
1545         if (was_opened)
1546                 rndis_filter_close(nvdev);
1547
1548         rndis_filter_device_remove(hdev, nvdev);
1549
1550         nvdev = rndis_filter_device_add(hdev, &device_info);
1551         if (IS_ERR(nvdev)) {
1552                 ret = PTR_ERR(nvdev);
1553
1554                 device_info.send_sections = orig.tx_pending;
1555                 device_info.recv_sections = orig.rx_pending;
1556                 nvdev = rndis_filter_device_add(hdev, &device_info);
1557                 if (IS_ERR(nvdev)) {
1558                         netdev_err(ndev, "restoring ringparam failed: %ld\n",
1559                                    PTR_ERR(nvdev));
1560                         return ret;
1561                 }
1562         }
1563
1564         if (was_opened)
1565                 rndis_filter_open(nvdev);
1566         netif_device_attach(ndev);
1567
1568         /* We may have missed link change notifications */
1569         ndevctx->last_reconfig = 0;
1570         schedule_delayed_work(&ndevctx->dwork, 0);
1571
1572         return ret;
1573 }
1574
1575 static const struct ethtool_ops ethtool_ops = {
1576         .get_drvinfo    = netvsc_get_drvinfo,
1577         .get_link       = ethtool_op_get_link,
1578         .get_ethtool_stats = netvsc_get_ethtool_stats,
1579         .get_sset_count = netvsc_get_sset_count,
1580         .get_strings    = netvsc_get_strings,
1581         .get_channels   = netvsc_get_channels,
1582         .set_channels   = netvsc_set_channels,
1583         .get_ts_info    = ethtool_op_get_ts_info,
1584         .get_rxnfc      = netvsc_get_rxnfc,
1585         .set_rxnfc      = netvsc_set_rxnfc,
1586         .get_rxfh_key_size = netvsc_get_rxfh_key_size,
1587         .get_rxfh_indir_size = netvsc_rss_indir_size,
1588         .get_rxfh       = netvsc_get_rxfh,
1589         .set_rxfh       = netvsc_set_rxfh,
1590         .get_link_ksettings = netvsc_get_link_ksettings,
1591         .set_link_ksettings = netvsc_set_link_ksettings,
1592         .get_ringparam  = netvsc_get_ringparam,
1593         .set_ringparam  = netvsc_set_ringparam,
1594 };
1595
1596 static const struct net_device_ops device_ops = {
1597         .ndo_open =                     netvsc_open,
1598         .ndo_stop =                     netvsc_close,
1599         .ndo_start_xmit =               netvsc_start_xmit,
1600         .ndo_set_rx_mode =              netvsc_set_multicast_list,
1601         .ndo_change_mtu =               netvsc_change_mtu,
1602         .ndo_validate_addr =            eth_validate_addr,
1603         .ndo_set_mac_address =          netvsc_set_mac_addr,
1604         .ndo_select_queue =             netvsc_select_queue,
1605         .ndo_get_stats64 =              netvsc_get_stats64,
1606 #ifdef CONFIG_NET_POLL_CONTROLLER
1607         .ndo_poll_controller =          netvsc_poll_controller,
1608 #endif
1609 };
1610
1611 /*
1612  * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
1613  * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
1614  * present send GARP packet to network peers with netif_notify_peers().
1615  */
1616 static void netvsc_link_change(struct work_struct *w)
1617 {
1618         struct net_device_context *ndev_ctx =
1619                 container_of(w, struct net_device_context, dwork.work);
1620         struct hv_device *device_obj = ndev_ctx->device_ctx;
1621         struct net_device *net = hv_get_drvdata(device_obj);
1622         struct netvsc_device *net_device;
1623         struct rndis_device *rdev;
1624         struct netvsc_reconfig *event = NULL;
1625         bool notify = false, reschedule = false;
1626         unsigned long flags, next_reconfig, delay;
1627
1628         /* if changes are happening, comeback later */
1629         if (!rtnl_trylock()) {
1630                 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1631                 return;
1632         }
1633
1634         net_device = rtnl_dereference(ndev_ctx->nvdev);
1635         if (!net_device)
1636                 goto out_unlock;
1637
1638         rdev = net_device->extension;
1639
1640         next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT;
1641         if (time_is_after_jiffies(next_reconfig)) {
1642                 /* link_watch only sends one notification with current state
1643                  * per second, avoid doing reconfig more frequently. Handle
1644                  * wrap around.
1645                  */
1646                 delay = next_reconfig - jiffies;
1647                 delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT;
1648                 schedule_delayed_work(&ndev_ctx->dwork, delay);
1649                 goto out_unlock;
1650         }
1651         ndev_ctx->last_reconfig = jiffies;
1652
1653         spin_lock_irqsave(&ndev_ctx->lock, flags);
1654         if (!list_empty(&ndev_ctx->reconfig_events)) {
1655                 event = list_first_entry(&ndev_ctx->reconfig_events,
1656                                          struct netvsc_reconfig, list);
1657                 list_del(&event->list);
1658                 reschedule = !list_empty(&ndev_ctx->reconfig_events);
1659         }
1660         spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1661
1662         if (!event)
1663                 goto out_unlock;
1664
1665         switch (event->event) {
1666                 /* Only the following events are possible due to the check in
1667                  * netvsc_linkstatus_callback()
1668                  */
1669         case RNDIS_STATUS_MEDIA_CONNECT:
1670                 if (rdev->link_state) {
1671                         rdev->link_state = false;
1672                         netif_carrier_on(net);
1673                         netif_tx_wake_all_queues(net);
1674                 } else {
1675                         notify = true;
1676                 }
1677                 kfree(event);
1678                 break;
1679         case RNDIS_STATUS_MEDIA_DISCONNECT:
1680                 if (!rdev->link_state) {
1681                         rdev->link_state = true;
1682                         netif_carrier_off(net);
1683                         netif_tx_stop_all_queues(net);
1684                 }
1685                 kfree(event);
1686                 break;
1687         case RNDIS_STATUS_NETWORK_CHANGE:
1688                 /* Only makes sense if carrier is present */
1689                 if (!rdev->link_state) {
1690                         rdev->link_state = true;
1691                         netif_carrier_off(net);
1692                         netif_tx_stop_all_queues(net);
1693                         event->event = RNDIS_STATUS_MEDIA_CONNECT;
1694                         spin_lock_irqsave(&ndev_ctx->lock, flags);
1695                         list_add(&event->list, &ndev_ctx->reconfig_events);
1696                         spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1697                         reschedule = true;
1698                 }
1699                 break;
1700         }
1701
1702         rtnl_unlock();
1703
1704         if (notify)
1705                 netdev_notify_peers(net);
1706
1707         /* link_watch only sends one notification with current state per
1708          * second, handle next reconfig event in 2 seconds.
1709          */
1710         if (reschedule)
1711                 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1712
1713         return;
1714
1715 out_unlock:
1716         rtnl_unlock();
1717 }
1718
1719 static struct net_device *get_netvsc_bymac(const u8 *mac)
1720 {
1721         struct net_device *dev;
1722
1723         ASSERT_RTNL();
1724
1725         for_each_netdev(&init_net, dev) {
1726                 if (dev->netdev_ops != &device_ops)
1727                         continue;       /* not a netvsc device */
1728
1729                 if (ether_addr_equal(mac, dev->perm_addr))
1730                         return dev;
1731         }
1732
1733         return NULL;
1734 }
1735
1736 static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
1737 {
1738         struct net_device *dev;
1739
1740         ASSERT_RTNL();
1741
1742         for_each_netdev(&init_net, dev) {
1743                 struct net_device_context *net_device_ctx;
1744
1745                 if (dev->netdev_ops != &device_ops)
1746                         continue;       /* not a netvsc device */
1747
1748                 net_device_ctx = netdev_priv(dev);
1749                 if (!rtnl_dereference(net_device_ctx->nvdev))
1750                         continue;       /* device is removed */
1751
1752                 if (rtnl_dereference(net_device_ctx->vf_netdev) == vf_netdev)
1753                         return dev;     /* a match */
1754         }
1755
1756         return NULL;
1757 }
1758
1759 /* Called when VF is injecting data into network stack.
1760  * Change the associated network device from VF to netvsc.
1761  * note: already called with rcu_read_lock
1762  */
1763 static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb)
1764 {
1765         struct sk_buff *skb = *pskb;
1766         struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data);
1767         struct net_device_context *ndev_ctx = netdev_priv(ndev);
1768         struct netvsc_vf_pcpu_stats *pcpu_stats
1769                  = this_cpu_ptr(ndev_ctx->vf_stats);
1770
1771         skb->dev = ndev;
1772
1773         u64_stats_update_begin(&pcpu_stats->syncp);
1774         pcpu_stats->rx_packets++;
1775         pcpu_stats->rx_bytes += skb->len;
1776         u64_stats_update_end(&pcpu_stats->syncp);
1777
1778         return RX_HANDLER_ANOTHER;
1779 }
1780
1781 static int netvsc_vf_join(struct net_device *vf_netdev,
1782                           struct net_device *ndev)
1783 {
1784         struct net_device_context *ndev_ctx = netdev_priv(ndev);
1785         int ret;
1786
1787         ret = netdev_rx_handler_register(vf_netdev,
1788                                          netvsc_vf_handle_frame, ndev);
1789         if (ret != 0) {
1790                 netdev_err(vf_netdev,
1791                            "can not register netvsc VF receive handler (err = %d)\n",
1792                            ret);
1793                 goto rx_handler_failed;
1794         }
1795
1796         ret = netdev_upper_dev_link(vf_netdev, ndev, NULL);
1797         if (ret != 0) {
1798                 netdev_err(vf_netdev,
1799                            "can not set master device %s (err = %d)\n",
1800                            ndev->name, ret);
1801                 goto upper_link_failed;
1802         }
1803
1804         /* set slave flag before open to prevent IPv6 addrconf */
1805         vf_netdev->flags |= IFF_SLAVE;
1806
1807         schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
1808
1809         call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
1810
1811         netdev_info(vf_netdev, "joined to %s\n", ndev->name);
1812         return 0;
1813
1814 upper_link_failed:
1815         netdev_rx_handler_unregister(vf_netdev);
1816 rx_handler_failed:
1817         return ret;
1818 }
1819
1820 static void __netvsc_vf_setup(struct net_device *ndev,
1821                               struct net_device *vf_netdev)
1822 {
1823         int ret;
1824
1825         /* Align MTU of VF with master */
1826         ret = dev_set_mtu(vf_netdev, ndev->mtu);
1827         if (ret)
1828                 netdev_warn(vf_netdev,
1829                             "unable to change mtu to %u\n", ndev->mtu);
1830
1831         if (netif_running(ndev)) {
1832                 ret = dev_open(vf_netdev);
1833                 if (ret)
1834                         netdev_warn(vf_netdev,
1835                                     "unable to open: %d\n", ret);
1836         }
1837 }
1838
1839 /* Setup VF as slave of the synthetic device.
1840  * Runs in workqueue to avoid recursion in netlink callbacks.
1841  */
1842 static void netvsc_vf_setup(struct work_struct *w)
1843 {
1844         struct net_device_context *ndev_ctx
1845                 = container_of(w, struct net_device_context, vf_takeover.work);
1846         struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx);
1847         struct net_device *vf_netdev;
1848
1849         if (!rtnl_trylock()) {
1850                 schedule_delayed_work(&ndev_ctx->vf_takeover, 0);
1851                 return;
1852         }
1853
1854         vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
1855         if (vf_netdev)
1856                 __netvsc_vf_setup(ndev, vf_netdev);
1857
1858         rtnl_unlock();
1859 }
1860
1861 static int netvsc_register_vf(struct net_device *vf_netdev)
1862 {
1863         struct net_device *ndev;
1864         struct net_device_context *net_device_ctx;
1865         struct netvsc_device *netvsc_dev;
1866
1867         if (vf_netdev->addr_len != ETH_ALEN)
1868                 return NOTIFY_DONE;
1869
1870         /*
1871          * We will use the MAC address to locate the synthetic interface to
1872          * associate with the VF interface. If we don't find a matching
1873          * synthetic interface, move on.
1874          */
1875         ndev = get_netvsc_bymac(vf_netdev->perm_addr);
1876         if (!ndev)
1877                 return NOTIFY_DONE;
1878
1879         net_device_ctx = netdev_priv(ndev);
1880         netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
1881         if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
1882                 return NOTIFY_DONE;
1883
1884         if (netvsc_vf_join(vf_netdev, ndev) != 0)
1885                 return NOTIFY_DONE;
1886
1887         netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
1888
1889         dev_hold(vf_netdev);
1890         rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
1891         return NOTIFY_OK;
1892 }
1893
1894 /* VF up/down change detected, schedule to change data path */
1895 static int netvsc_vf_changed(struct net_device *vf_netdev)
1896 {
1897         struct net_device_context *net_device_ctx;
1898         struct netvsc_device *netvsc_dev;
1899         struct net_device *ndev;
1900         bool vf_is_up = netif_running(vf_netdev);
1901
1902         ndev = get_netvsc_byref(vf_netdev);
1903         if (!ndev)
1904                 return NOTIFY_DONE;
1905
1906         net_device_ctx = netdev_priv(ndev);
1907         netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
1908         if (!netvsc_dev)
1909                 return NOTIFY_DONE;
1910
1911         netvsc_switch_datapath(ndev, vf_is_up);
1912         netdev_info(ndev, "Data path switched %s VF: %s\n",
1913                     vf_is_up ? "to" : "from", vf_netdev->name);
1914
1915         return NOTIFY_OK;
1916 }
1917
1918 static int netvsc_unregister_vf(struct net_device *vf_netdev)
1919 {
1920         struct net_device *ndev;
1921         struct net_device_context *net_device_ctx;
1922
1923         ndev = get_netvsc_byref(vf_netdev);
1924         if (!ndev)
1925                 return NOTIFY_DONE;
1926
1927         net_device_ctx = netdev_priv(ndev);
1928         cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
1929
1930         netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
1931
1932         netdev_rx_handler_unregister(vf_netdev);
1933         netdev_upper_dev_unlink(vf_netdev, ndev);
1934         RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
1935         dev_put(vf_netdev);
1936
1937         return NOTIFY_OK;
1938 }
1939
1940 static int netvsc_probe(struct hv_device *dev,
1941                         const struct hv_vmbus_device_id *dev_id)
1942 {
1943         struct net_device *net = NULL;
1944         struct net_device_context *net_device_ctx;
1945         struct netvsc_device_info device_info;
1946         struct netvsc_device *nvdev;
1947         int ret = -ENOMEM;
1948
1949         net = alloc_etherdev_mq(sizeof(struct net_device_context),
1950                                 VRSS_CHANNEL_MAX);
1951         if (!net)
1952                 goto no_net;
1953
1954         netif_carrier_off(net);
1955
1956         netvsc_init_settings(net);
1957
1958         net_device_ctx = netdev_priv(net);
1959         net_device_ctx->device_ctx = dev;
1960         net_device_ctx->msg_enable = netif_msg_init(debug, default_msg);
1961         if (netif_msg_probe(net_device_ctx))
1962                 netdev_dbg(net, "netvsc msg_enable: %d\n",
1963                            net_device_ctx->msg_enable);
1964
1965         hv_set_drvdata(dev, net);
1966
1967         INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change);
1968
1969         spin_lock_init(&net_device_ctx->lock);
1970         INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
1971         INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup);
1972
1973         net_device_ctx->vf_stats
1974                 = netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats);
1975         if (!net_device_ctx->vf_stats)
1976                 goto no_stats;
1977
1978         net->netdev_ops = &device_ops;
1979         net->ethtool_ops = &ethtool_ops;
1980         SET_NETDEV_DEV(net, &dev->device);
1981
1982         /* We always need headroom for rndis header */
1983         net->needed_headroom = RNDIS_AND_PPI_SIZE;
1984
1985         /* Initialize the number of queues to be 1, we may change it if more
1986          * channels are offered later.
1987          */
1988         netif_set_real_num_tx_queues(net, 1);
1989         netif_set_real_num_rx_queues(net, 1);
1990
1991         /* Notify the netvsc driver of the new device */
1992         memset(&device_info, 0, sizeof(device_info));
1993         device_info.num_chn = VRSS_CHANNEL_DEFAULT;
1994         device_info.send_sections = NETVSC_DEFAULT_TX;
1995         device_info.send_section_size = NETVSC_SEND_SECTION_SIZE;
1996         device_info.recv_sections = NETVSC_DEFAULT_RX;
1997         device_info.recv_section_size = NETVSC_RECV_SECTION_SIZE;
1998
1999         nvdev = rndis_filter_device_add(dev, &device_info);
2000         if (IS_ERR(nvdev)) {
2001                 ret = PTR_ERR(nvdev);
2002                 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
2003                 goto rndis_failed;
2004         }
2005
2006         memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN);
2007
2008         /* hw_features computed in rndis_netdev_set_hwcaps() */
2009         net->features = net->hw_features |
2010                 NETIF_F_HIGHDMA | NETIF_F_SG |
2011                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
2012         net->vlan_features = net->features;
2013
2014         netdev_lockdep_set_classes(net);
2015
2016         /* MTU range: 68 - 1500 or 65521 */
2017         net->min_mtu = NETVSC_MTU_MIN;
2018         if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
2019                 net->max_mtu = NETVSC_MTU - ETH_HLEN;
2020         else
2021                 net->max_mtu = ETH_DATA_LEN;
2022
2023         ret = register_netdev(net);
2024         if (ret != 0) {
2025                 pr_err("Unable to register netdev.\n");
2026                 goto register_failed;
2027         }
2028
2029         return ret;
2030
2031 register_failed:
2032         rndis_filter_device_remove(dev, nvdev);
2033 rndis_failed:
2034         free_percpu(net_device_ctx->vf_stats);
2035 no_stats:
2036         hv_set_drvdata(dev, NULL);
2037         free_netdev(net);
2038 no_net:
2039         return ret;
2040 }
2041
2042 static int netvsc_remove(struct hv_device *dev)
2043 {
2044         struct net_device_context *ndev_ctx;
2045         struct net_device *vf_netdev;
2046         struct net_device *net;
2047
2048         net = hv_get_drvdata(dev);
2049         if (net == NULL) {
2050                 dev_err(&dev->device, "No net device to remove\n");
2051                 return 0;
2052         }
2053
2054         ndev_ctx = netdev_priv(net);
2055
2056         netif_device_detach(net);
2057
2058         cancel_delayed_work_sync(&ndev_ctx->dwork);
2059
2060         /*
2061          * Call to the vsc driver to let it know that the device is being
2062          * removed. Also blocks mtu and channel changes.
2063          */
2064         rtnl_lock();
2065         vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
2066         if (vf_netdev)
2067                 netvsc_unregister_vf(vf_netdev);
2068
2069         unregister_netdevice(net);
2070
2071         rndis_filter_device_remove(dev,
2072                                    rtnl_dereference(ndev_ctx->nvdev));
2073         rtnl_unlock();
2074
2075         hv_set_drvdata(dev, NULL);
2076
2077         free_percpu(ndev_ctx->vf_stats);
2078         free_netdev(net);
2079         return 0;
2080 }
2081
2082 static const struct hv_vmbus_device_id id_table[] = {
2083         /* Network guid */
2084         { HV_NIC_GUID, },
2085         { },
2086 };
2087
2088 MODULE_DEVICE_TABLE(vmbus, id_table);
2089
2090 /* The one and only one */
2091 static struct  hv_driver netvsc_drv = {
2092         .name = KBUILD_MODNAME,
2093         .id_table = id_table,
2094         .probe = netvsc_probe,
2095         .remove = netvsc_remove,
2096 };
2097
2098 /*
2099  * On Hyper-V, every VF interface is matched with a corresponding
2100  * synthetic interface. The synthetic interface is presented first
2101  * to the guest. When the corresponding VF instance is registered,
2102  * we will take care of switching the data path.
2103  */
2104 static int netvsc_netdev_event(struct notifier_block *this,
2105                                unsigned long event, void *ptr)
2106 {
2107         struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
2108
2109         /* Skip our own events */
2110         if (event_dev->netdev_ops == &device_ops)
2111                 return NOTIFY_DONE;
2112
2113         /* Avoid non-Ethernet type devices */
2114         if (event_dev->type != ARPHRD_ETHER)
2115                 return NOTIFY_DONE;
2116
2117         /* Avoid Vlan dev with same MAC registering as VF */
2118         if (is_vlan_dev(event_dev))
2119                 return NOTIFY_DONE;
2120
2121         /* Avoid Bonding master dev with same MAC registering as VF */
2122         if ((event_dev->priv_flags & IFF_BONDING) &&
2123             (event_dev->flags & IFF_MASTER))
2124                 return NOTIFY_DONE;
2125
2126         switch (event) {
2127         case NETDEV_REGISTER:
2128                 return netvsc_register_vf(event_dev);
2129         case NETDEV_UNREGISTER:
2130                 return netvsc_unregister_vf(event_dev);
2131         case NETDEV_UP:
2132         case NETDEV_DOWN:
2133                 return netvsc_vf_changed(event_dev);
2134         default:
2135                 return NOTIFY_DONE;
2136         }
2137 }
2138
2139 static struct notifier_block netvsc_netdev_notifier = {
2140         .notifier_call = netvsc_netdev_event,
2141 };
2142
2143 static void __exit netvsc_drv_exit(void)
2144 {
2145         unregister_netdevice_notifier(&netvsc_netdev_notifier);
2146         vmbus_driver_unregister(&netvsc_drv);
2147 }
2148
2149 static int __init netvsc_drv_init(void)
2150 {
2151         int ret;
2152
2153         if (ring_size < RING_SIZE_MIN) {
2154                 ring_size = RING_SIZE_MIN;
2155                 pr_info("Increased ring_size to %u (min allowed)\n",
2156                         ring_size);
2157         }
2158         netvsc_ring_bytes = ring_size * PAGE_SIZE;
2159         netvsc_ring_reciprocal = reciprocal_value(netvsc_ring_bytes);
2160
2161         ret = vmbus_driver_register(&netvsc_drv);
2162         if (ret)
2163                 return ret;
2164
2165         register_netdevice_notifier(&netvsc_netdev_notifier);
2166         return 0;
2167 }
2168
2169 MODULE_LICENSE("GPL");
2170 MODULE_DESCRIPTION("Microsoft Hyper-V network driver");
2171
2172 module_init(netvsc_drv_init);
2173 module_exit(netvsc_drv_exit);