drivers/net/ethernet/hisilicon/hns3/hns3_enet.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright (c) 2016-2017 Hisilicon Limited.
   3
   4 #include <linux/dma-mapping.h>
   5 #include <linux/etherdevice.h>
   6 #include <linux/interrupt.h>
   7 #ifdef CONFIG_RFS_ACCEL
   8 #include <linux/cpu_rmap.h>
   9 #endif
  10 #include <linux/if_vlan.h>
  11 #include <linux/irq.h>
  12 #include <linux/ip.h>
  13 #include <linux/ipv6.h>
  14 #include <linux/module.h>
  15 #include <linux/pci.h>
  16 #include <linux/aer.h>
  17 #include <linux/skbuff.h>
  18 #include <linux/sctp.h>
  19 #include <net/gre.h>
  20 #include <net/ip6_checksum.h>
  21 #include <net/pkt_cls.h>
  22 #include <net/tcp.h>
  23 #include <net/vxlan.h>
  24 #include <net/geneve.h>
  25
  26 #include "hnae3.h"
  27 #include "hns3_enet.h"
  28 /* All hns3 tracepoints are defined by the include below, which
  29  * must be included exactly once across the whole kernel with
  30  * CREATE_TRACE_POINTS defined
  31  */
  32 #define CREATE_TRACE_POINTS
  33 #include "hns3_trace.h"
  34
  35 #define hns3_set_field(origin, shift, val)      ((origin) |= ((val) << (shift)))
  36 #define hns3_tx_bd_count(S)     DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE)
  37
  38 #define hns3_rl_err(fmt, ...)                                           \
  39         do {                                                            \
  40                 if (net_ratelimit())                                    \
  41                         netdev_err(fmt, ##__VA_ARGS__);                 \
  42         } while (0)
  43
  44 static void hns3_clear_all_ring(struct hnae3_handle *h, bool force);
  45
  46 static const char hns3_driver_name[] = "hns3";
  47 static const char hns3_driver_string[] =
  48                         "Hisilicon Ethernet Network Driver for Hip08 Family";
  49 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
  50 static struct hnae3_client client;
  51
  52 static int debug = -1;
  53 module_param(debug, int, 0);
  54 MODULE_PARM_DESC(debug, " Network interface message level setting");
  55
  56 #define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  57                            NETIF_MSG_IFDOWN | NETIF_MSG_IFUP)
  58
  59 #define HNS3_INNER_VLAN_TAG     1
  60 #define HNS3_OUTER_VLAN_TAG     2
  61
  62 #define HNS3_MIN_TX_LEN         33U
  63
  64 /* hns3_pci_tbl - PCI Device ID Table
  65  *
  66  * Last entry must be all 0s
  67  *
  68  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  69  *   Class, Class Mask, private data (not used) }
  70  */
  71 static const struct pci_device_id hns3_pci_tbl[] = {
  72         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
  73         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
  74         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
  75          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  76         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
  77          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  78         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
  79          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  80         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
  81          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  82         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
  83          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  84         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
  85         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
  86          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  87         /* required last entry */
  88         {0, }
  89 };
  90 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
  91
  92 static irqreturn_t hns3_irq_handle(int irq, void *vector)
  93 {
  94         struct hns3_enet_tqp_vector *tqp_vector = vector;
  95
  96         napi_schedule_irqoff(&tqp_vector->napi);
  97
  98         return IRQ_HANDLED;
  99 }
 100
 101 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
 102 {
 103         struct hns3_enet_tqp_vector *tqp_vectors;
 104         unsigned int i;
 105
 106         for (i = 0; i < priv->vector_num; i++) {
 107                 tqp_vectors = &priv->tqp_vector[i];
 108
 109                 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
 110                         continue;
 111
 112                 /* clear the affinity mask */
 113                 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
 114
 115                 /* release the irq resource */
 116                 free_irq(tqp_vectors->vector_irq, tqp_vectors);
 117                 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
 118         }
 119 }
 120
 121 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
 122 {
 123         struct hns3_enet_tqp_vector *tqp_vectors;
 124         int txrx_int_idx = 0;
 125         int rx_int_idx = 0;
 126         int tx_int_idx = 0;
 127         unsigned int i;
 128         int ret;
 129
 130         for (i = 0; i < priv->vector_num; i++) {
 131                 tqp_vectors = &priv->tqp_vector[i];
 132
 133                 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
 134                         continue;
 135
 136                 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
 137                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 138                                  "%s-%s-%s-%d", hns3_driver_name,
 139                                  pci_name(priv->ae_handle->pdev),
 140                                  "TxRx", txrx_int_idx++);
 141                         txrx_int_idx++;
 142                 } else if (tqp_vectors->rx_group.ring) {
 143                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 144                                  "%s-%s-%s-%d", hns3_driver_name,
 145                                  pci_name(priv->ae_handle->pdev),
 146                                  "Rx", rx_int_idx++);
 147                 } else if (tqp_vectors->tx_group.ring) {
 148                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 149                                  "%s-%s-%s-%d", hns3_driver_name,
 150                                  pci_name(priv->ae_handle->pdev),
 151                                  "Tx", tx_int_idx++);
 152                 } else {
 153                         /* Skip this unused q_vector */
 154                         continue;
 155                 }
 156
 157                 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
 158
 159                 irq_set_status_flags(tqp_vectors->vector_irq, IRQ_NOAUTOEN);
 160                 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
 161                                   tqp_vectors->name, tqp_vectors);
 162                 if (ret) {
 163                         netdev_err(priv->netdev, "request irq(%d) fail\n",
 164                                    tqp_vectors->vector_irq);
 165                         hns3_nic_uninit_irq(priv);
 166                         return ret;
 167                 }
 168
 169                 irq_set_affinity_hint(tqp_vectors->vector_irq,
 170                                       &tqp_vectors->affinity_mask);
 171
 172                 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
 173         }
 174
 175         return 0;
 176 }
 177
 178 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
 179                                  u32 mask_en)
 180 {
 181         writel(mask_en, tqp_vector->mask_addr);
 182 }
 183
 184 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
 185 {
 186         napi_enable(&tqp_vector->napi);
 187         enable_irq(tqp_vector->vector_irq);
 188
 189         /* enable vector */
 190         hns3_mask_vector_irq(tqp_vector, 1);
 191 }
 192
 193 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 194 {
 195         /* disable vector */
 196         hns3_mask_vector_irq(tqp_vector, 0);
 197
 198         disable_irq(tqp_vector->vector_irq);
 199         napi_disable(&tqp_vector->napi);
 200 }
 201
 202 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
 203                                  u32 rl_value)
 204 {
 205         u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
 206
 207         /* this defines the configuration for RL (Interrupt Rate Limiter).
 208          * Rl defines rate of interrupts i.e. number of interrupts-per-second
 209          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 210          */
 211
 212         if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
 213             !tqp_vector->rx_group.coal.gl_adapt_enable)
 214                 /* According to the hardware, the range of rl_reg is
 215                  * 0-59 and the unit is 4.
 216                  */
 217                 rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
 218
 219         writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
 220 }
 221
 222 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 223                                     u32 gl_value)
 224 {
 225         u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
 226
 227         writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
 228 }
 229
 230 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 231                                     u32 gl_value)
 232 {
 233         u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
 234
 235         writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 236 }
 237
 238 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
 239                                    struct hns3_nic_priv *priv)
 240 {
 241         /* initialize the configuration for interrupt coalescing.
 242          * 1. GL (Interrupt Gap Limiter)
 243          * 2. RL (Interrupt Rate Limiter)
 244          *
 245          * Default: enable interrupt coalescing self-adaptive and GL
 246          */
 247         tqp_vector->tx_group.coal.gl_adapt_enable = 1;
 248         tqp_vector->rx_group.coal.gl_adapt_enable = 1;
 249
 250         tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
 251         tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
 252
 253         tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
 254         tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
 255 }
 256
 257 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
 258                                       struct hns3_nic_priv *priv)
 259 {
 260         struct hnae3_handle *h = priv->ae_handle;
 261
 262         hns3_set_vector_coalesce_tx_gl(tqp_vector,
 263                                        tqp_vector->tx_group.coal.int_gl);
 264         hns3_set_vector_coalesce_rx_gl(tqp_vector,
 265                                        tqp_vector->rx_group.coal.int_gl);
 266         hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
 267 }
 268
 269 static int hns3_nic_set_real_num_queue(struct net_device *netdev)
 270 {
 271         struct hnae3_handle *h = hns3_get_handle(netdev);
 272         struct hnae3_knic_private_info *kinfo = &h->kinfo;
 273         unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
 274         int i, ret;
 275
 276         if (kinfo->num_tc <= 1) {
 277                 netdev_reset_tc(netdev);
 278         } else {
 279                 ret = netdev_set_num_tc(netdev, kinfo->num_tc);
 280                 if (ret) {
 281                         netdev_err(netdev,
 282                                    "netdev_set_num_tc fail, ret=%d!\n", ret);
 283                         return ret;
 284                 }
 285
 286                 for (i = 0; i < HNAE3_MAX_TC; i++) {
 287                         if (!kinfo->tc_info[i].enable)
 288                                 continue;
 289
 290                         netdev_set_tc_queue(netdev,
 291                                             kinfo->tc_info[i].tc,
 292                                             kinfo->tc_info[i].tqp_count,
 293                                             kinfo->tc_info[i].tqp_offset);
 294                 }
 295         }
 296
 297         ret = netif_set_real_num_tx_queues(netdev, queue_size);
 298         if (ret) {
 299                 netdev_err(netdev,
 300                            "netif_set_real_num_tx_queues fail, ret=%d!\n", ret);
 301                 return ret;
 302         }
 303
 304         ret = netif_set_real_num_rx_queues(netdev, queue_size);
 305         if (ret) {
 306                 netdev_err(netdev,
 307                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
 308                 return ret;
 309         }
 310
 311         return 0;
 312 }
 313
 314 static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
 315 {
 316         u16 alloc_tqps, max_rss_size, rss_size;
 317
 318         h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
 319         rss_size = alloc_tqps / h->kinfo.num_tc;
 320
 321         return min_t(u16, rss_size, max_rss_size);
 322 }
 323
 324 static void hns3_tqp_enable(struct hnae3_queue *tqp)
 325 {
 326         u32 rcb_reg;
 327
 328         rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
 329         rcb_reg |= BIT(HNS3_RING_EN_B);
 330         hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
 331 }
 332
 333 static void hns3_tqp_disable(struct hnae3_queue *tqp)
 334 {
 335         u32 rcb_reg;
 336
 337         rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
 338         rcb_reg &= ~BIT(HNS3_RING_EN_B);
 339         hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
 340 }
 341
 342 static void hns3_free_rx_cpu_rmap(struct net_device *netdev)
 343 {
 344 #ifdef CONFIG_RFS_ACCEL
 345         free_irq_cpu_rmap(netdev->rx_cpu_rmap);
 346         netdev->rx_cpu_rmap = NULL;
 347 #endif
 348 }
 349
 350 static int hns3_set_rx_cpu_rmap(struct net_device *netdev)
 351 {
 352 #ifdef CONFIG_RFS_ACCEL
 353         struct hns3_nic_priv *priv = netdev_priv(netdev);
 354         struct hns3_enet_tqp_vector *tqp_vector;
 355         int i, ret;
 356
 357         if (!netdev->rx_cpu_rmap) {
 358                 netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->vector_num);
 359                 if (!netdev->rx_cpu_rmap)
 360                         return -ENOMEM;
 361         }
 362
 363         for (i = 0; i < priv->vector_num; i++) {
 364                 tqp_vector = &priv->tqp_vector[i];
 365                 ret = irq_cpu_rmap_add(netdev->rx_cpu_rmap,
 366                                        tqp_vector->vector_irq);
 367                 if (ret) {
 368                         hns3_free_rx_cpu_rmap(netdev);
 369                         return ret;
 370                 }
 371         }
 372 #endif
 373         return 0;
 374 }
 375
 376 static int hns3_nic_net_up(struct net_device *netdev)
 377 {
 378         struct hns3_nic_priv *priv = netdev_priv(netdev);
 379         struct hnae3_handle *h = priv->ae_handle;
 380         int i, j;
 381         int ret;
 382
 383         ret = hns3_nic_reset_all_ring(h);
 384         if (ret)
 385                 return ret;
 386
 387         clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 388
 389         /* enable the vectors */
 390         for (i = 0; i < priv->vector_num; i++)
 391                 hns3_vector_enable(&priv->tqp_vector[i]);
 392
 393         /* enable rcb */
 394         for (j = 0; j < h->kinfo.num_tqps; j++)
 395                 hns3_tqp_enable(h->kinfo.tqp[j]);
 396
 397         /* start the ae_dev */
 398         ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
 399         if (ret) {
 400                 set_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 401                 while (j--)
 402                         hns3_tqp_disable(h->kinfo.tqp[j]);
 403
 404                 for (j = i - 1; j >= 0; j--)
 405                         hns3_vector_disable(&priv->tqp_vector[j]);
 406         }
 407
 408         return ret;
 409 }
 410
 411 static void hns3_config_xps(struct hns3_nic_priv *priv)
 412 {
 413         int i;
 414
 415         for (i = 0; i < priv->vector_num; i++) {
 416                 struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];
 417                 struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;
 418
 419                 while (ring) {
 420                         int ret;
 421
 422                         ret = netif_set_xps_queue(priv->netdev,
 423                                                   &tqp_vector->affinity_mask,
 424                                                   ring->tqp->tqp_index);
 425                         if (ret)
 426                                 netdev_warn(priv->netdev,
 427                                             "set xps queue failed: %d", ret);
 428
 429                         ring = ring->next;
 430                 }
 431         }
 432 }
 433
 434 static int hns3_nic_net_open(struct net_device *netdev)
 435 {
 436         struct hns3_nic_priv *priv = netdev_priv(netdev);
 437         struct hnae3_handle *h = hns3_get_handle(netdev);
 438         struct hnae3_knic_private_info *kinfo;
 439         int i, ret;
 440
 441         if (hns3_nic_resetting(netdev))
 442                 return -EBUSY;
 443
 444         netif_carrier_off(netdev);
 445
 446         ret = hns3_nic_set_real_num_queue(netdev);
 447         if (ret)
 448                 return ret;
 449
 450         ret = hns3_nic_net_up(netdev);
 451         if (ret) {
 452                 netdev_err(netdev, "net up fail, ret=%d!\n", ret);
 453                 return ret;
 454         }
 455
 456         kinfo = &h->kinfo;
 457         for (i = 0; i < HNAE3_MAX_USER_PRIO; i++)
 458                 netdev_set_prio_tc_map(netdev, i, kinfo->prio_tc[i]);
 459
 460         if (h->ae_algo->ops->set_timer_task)
 461                 h->ae_algo->ops->set_timer_task(priv->ae_handle, true);
 462
 463         hns3_config_xps(priv);
 464
 465         netif_dbg(h, drv, netdev, "net open\n");
 466
 467         return 0;
 468 }
 469
 470 static void hns3_reset_tx_queue(struct hnae3_handle *h)
 471 {
 472         struct net_device *ndev = h->kinfo.netdev;
 473         struct hns3_nic_priv *priv = netdev_priv(ndev);
 474         struct netdev_queue *dev_queue;
 475         u32 i;
 476
 477         for (i = 0; i < h->kinfo.num_tqps; i++) {
 478                 dev_queue = netdev_get_tx_queue(ndev,
 479                                                 priv->ring[i].queue_index);
 480                 netdev_tx_reset_queue(dev_queue);
 481         }
 482 }
 483
 484 static void hns3_nic_net_down(struct net_device *netdev)
 485 {
 486         struct hns3_nic_priv *priv = netdev_priv(netdev);
 487         struct hnae3_handle *h = hns3_get_handle(netdev);
 488         const struct hnae3_ae_ops *ops;
 489         int i;
 490
 491         /* disable vectors */
 492         for (i = 0; i < priv->vector_num; i++)
 493                 hns3_vector_disable(&priv->tqp_vector[i]);
 494
 495         /* disable rcb */
 496         for (i = 0; i < h->kinfo.num_tqps; i++)
 497                 hns3_tqp_disable(h->kinfo.tqp[i]);
 498
 499         /* stop ae_dev */
 500         ops = priv->ae_handle->ae_algo->ops;
 501         if (ops->stop)
 502                 ops->stop(priv->ae_handle);
 503
 504         /* delay ring buffer clearing to hns3_reset_notify_uninit_enet
 505          * during reset process, because driver may not be able
 506          * to disable the ring through firmware when downing the netdev.
 507          */
 508         if (!hns3_nic_resetting(netdev))
 509                 hns3_clear_all_ring(priv->ae_handle, false);
 510
 511         hns3_reset_tx_queue(priv->ae_handle);
 512 }
 513
 514 static int hns3_nic_net_stop(struct net_device *netdev)
 515 {
 516         struct hns3_nic_priv *priv = netdev_priv(netdev);
 517         struct hnae3_handle *h = hns3_get_handle(netdev);
 518
 519         if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
 520                 return 0;
 521
 522         netif_dbg(h, drv, netdev, "net stop\n");
 523
 524         if (h->ae_algo->ops->set_timer_task)
 525                 h->ae_algo->ops->set_timer_task(priv->ae_handle, false);
 526
 527         netif_tx_stop_all_queues(netdev);
 528         netif_carrier_off(netdev);
 529
 530         hns3_nic_net_down(netdev);
 531
 532         return 0;
 533 }
 534
 535 static int hns3_nic_uc_sync(struct net_device *netdev,
 536                             const unsigned char *addr)
 537 {
 538         struct hnae3_handle *h = hns3_get_handle(netdev);
 539
 540         if (h->ae_algo->ops->add_uc_addr)
 541                 return h->ae_algo->ops->add_uc_addr(h, addr);
 542
 543         return 0;
 544 }
 545
 546 static int hns3_nic_uc_unsync(struct net_device *netdev,
 547                               const unsigned char *addr)
 548 {
 549         struct hnae3_handle *h = hns3_get_handle(netdev);
 550
 551         /* need ignore the request of removing device address, because
 552          * we store the device address and other addresses of uc list
 553          * in the function's mac filter list.
 554          */
 555         if (ether_addr_equal(addr, netdev->dev_addr))
 556                 return 0;
 557
 558         if (h->ae_algo->ops->rm_uc_addr)
 559                 return h->ae_algo->ops->rm_uc_addr(h, addr);
 560
 561         return 0;
 562 }
 563
 564 static int hns3_nic_mc_sync(struct net_device *netdev,
 565                             const unsigned char *addr)
 566 {
 567         struct hnae3_handle *h = hns3_get_handle(netdev);
 568
 569         if (h->ae_algo->ops->add_mc_addr)
 570                 return h->ae_algo->ops->add_mc_addr(h, addr);
 571
 572         return 0;
 573 }
 574
 575 static int hns3_nic_mc_unsync(struct net_device *netdev,
 576                               const unsigned char *addr)
 577 {
 578         struct hnae3_handle *h = hns3_get_handle(netdev);
 579
 580         if (h->ae_algo->ops->rm_mc_addr)
 581                 return h->ae_algo->ops->rm_mc_addr(h, addr);
 582
 583         return 0;
 584 }
 585
 586 static u8 hns3_get_netdev_flags(struct net_device *netdev)
 587 {
 588         u8 flags = 0;
 589
 590         if (netdev->flags & IFF_PROMISC) {
 591                 flags = HNAE3_USER_UPE | HNAE3_USER_MPE | HNAE3_BPE;
 592         } else {
 593                 flags |= HNAE3_VLAN_FLTR;
 594                 if (netdev->flags & IFF_ALLMULTI)
 595                         flags |= HNAE3_USER_MPE;
 596         }
 597
 598         return flags;
 599 }
 600
 601 static void hns3_nic_set_rx_mode(struct net_device *netdev)
 602 {
 603         struct hnae3_handle *h = hns3_get_handle(netdev);
 604         u8 new_flags;
 605
 606         new_flags = hns3_get_netdev_flags(netdev);
 607
 608         __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
 609         __dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync);
 610
 611         /* User mode Promisc mode enable and vlan filtering is disabled to
 612          * let all packets in.
 613          */
 614         h->netdev_flags = new_flags;
 615         hns3_request_update_promisc_mode(h);
 616 }
 617
 618 void hns3_request_update_promisc_mode(struct hnae3_handle *handle)
 619 {
 620         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 621
 622         if (ops->request_update_promisc_mode)
 623                 ops->request_update_promisc_mode(handle);
 624 }
 625
 626 void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
 627 {
 628         struct hns3_nic_priv *priv = netdev_priv(netdev);
 629         struct hnae3_handle *h = priv->ae_handle;
 630         bool last_state;
 631
 632         if (h->pdev->revision >= 0x21 && h->ae_algo->ops->enable_vlan_filter) {
 633                 last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
 634                 if (enable != last_state) {
 635                         netdev_info(netdev,
 636                                     "%s vlan filter\n",
 637                                     enable ? "enable" : "disable");
 638                         h->ae_algo->ops->enable_vlan_filter(h, enable);
 639                 }
 640         }
 641 }
 642
 643 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
 644                         u16 *mss, u32 *type_cs_vlan_tso)
 645 {
 646         u32 l4_offset, hdr_len;
 647         union l3_hdr_info l3;
 648         union l4_hdr_info l4;
 649         u32 l4_paylen;
 650         int ret;
 651
 652         if (!skb_is_gso(skb))
 653                 return 0;
 654
 655         ret = skb_cow_head(skb, 0);
 656         if (unlikely(ret < 0))
 657                 return ret;
 658
 659         l3.hdr = skb_network_header(skb);
 660         l4.hdr = skb_transport_header(skb);
 661
 662         /* Software should clear the IPv4's checksum field when tso is
 663          * needed.
 664          */
 665         if (l3.v4->version == 4)
 666                 l3.v4->check = 0;
 667
 668         /* tunnel packet */
 669         if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
 670                                          SKB_GSO_GRE_CSUM |
 671                                          SKB_GSO_UDP_TUNNEL |
 672                                          SKB_GSO_UDP_TUNNEL_CSUM)) {
 673                 if ((!(skb_shinfo(skb)->gso_type &
 674                     SKB_GSO_PARTIAL)) &&
 675                     (skb_shinfo(skb)->gso_type &
 676                     SKB_GSO_UDP_TUNNEL_CSUM)) {
 677                         /* Software should clear the udp's checksum
 678                          * field when tso is needed.
 679                          */
 680                         l4.udp->check = 0;
 681                 }
 682                 /* reset l3&l4 pointers from outer to inner headers */
 683                 l3.hdr = skb_inner_network_header(skb);
 684                 l4.hdr = skb_inner_transport_header(skb);
 685
 686                 /* Software should clear the IPv4's checksum field when
 687                  * tso is needed.
 688                  */
 689                 if (l3.v4->version == 4)
 690                         l3.v4->check = 0;
 691         }
 692
 693         /* normal or tunnel packet */
 694         l4_offset = l4.hdr - skb->data;
 695         hdr_len = (l4.tcp->doff << 2) + l4_offset;
 696
 697         /* remove payload length from inner pseudo checksum when tso */
 698         l4_paylen = skb->len - l4_offset;
 699         csum_replace_by_diff(&l4.tcp->check,
 700                              (__force __wsum)htonl(l4_paylen));
 701
 702         /* find the txbd field values */
 703         *paylen = skb->len - hdr_len;
 704         hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_TSO_B, 1);
 705
 706         /* get MSS for TSO */
 707         *mss = skb_shinfo(skb)->gso_size;
 708
 709         trace_hns3_tso(skb);
 710
 711         return 0;
 712 }
 713
 714 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
 715                                 u8 *il4_proto)
 716 {
 717         union l3_hdr_info l3;
 718         unsigned char *l4_hdr;
 719         unsigned char *exthdr;
 720         u8 l4_proto_tmp;
 721         __be16 frag_off;
 722
 723         /* find outer header point */
 724         l3.hdr = skb_network_header(skb);
 725         l4_hdr = skb_transport_header(skb);
 726
 727         if (skb->protocol == htons(ETH_P_IPV6)) {
 728                 exthdr = l3.hdr + sizeof(*l3.v6);
 729                 l4_proto_tmp = l3.v6->nexthdr;
 730                 if (l4_hdr != exthdr)
 731                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 732                                          &l4_proto_tmp, &frag_off);
 733         } else if (skb->protocol == htons(ETH_P_IP)) {
 734                 l4_proto_tmp = l3.v4->protocol;
 735         } else {
 736                 return -EINVAL;
 737         }
 738
 739         *ol4_proto = l4_proto_tmp;
 740
 741         /* tunnel packet */
 742         if (!skb->encapsulation) {
 743                 *il4_proto = 0;
 744                 return 0;
 745         }
 746
 747         /* find inner header point */
 748         l3.hdr = skb_inner_network_header(skb);
 749         l4_hdr = skb_inner_transport_header(skb);
 750
 751         if (l3.v6->version == 6) {
 752                 exthdr = l3.hdr + sizeof(*l3.v6);
 753                 l4_proto_tmp = l3.v6->nexthdr;
 754                 if (l4_hdr != exthdr)
 755                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 756                                          &l4_proto_tmp, &frag_off);
 757         } else if (l3.v4->version == 4) {
 758                 l4_proto_tmp = l3.v4->protocol;
 759         }
 760
 761         *il4_proto = l4_proto_tmp;
 762
 763         return 0;
 764 }
 765
 766 /* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
 767  * and it is udp packet, which has a dest port as the IANA assigned.
 768  * the hardware is expected to do the checksum offload, but the
 769  * hardware will not do the checksum offload when udp dest port is
 770  * 4789 or 6081.
 771  */
 772 static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
 773 {
 774         union l4_hdr_info l4;
 775
 776         l4.hdr = skb_transport_header(skb);
 777
 778         if (!(!skb->encapsulation &&
 779               (l4.udp->dest == htons(IANA_VXLAN_UDP_PORT) ||
 780               l4.udp->dest == htons(GENEVE_UDP_PORT))))
 781                 return false;
 782
 783         skb_checksum_help(skb);
 784
 785         return true;
 786 }
 787
 788 static void hns3_set_outer_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
 789                                   u32 *ol_type_vlan_len_msec)
 790 {
 791         u32 l2_len, l3_len, l4_len;
 792         unsigned char *il2_hdr;
 793         union l3_hdr_info l3;
 794         union l4_hdr_info l4;
 795
 796         l3.hdr = skb_network_header(skb);
 797         l4.hdr = skb_transport_header(skb);
 798
 799         /* compute OL2 header size, defined in 2 Bytes */
 800         l2_len = l3.hdr - skb->data;
 801         hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L2LEN_S, l2_len >> 1);
 802
 803         /* compute OL3 header size, defined in 4 Bytes */
 804         l3_len = l4.hdr - l3.hdr;
 805         hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_S, l3_len >> 2);
 806
 807         il2_hdr = skb_inner_mac_header(skb);
 808         /* compute OL4 header size, defined in 4 Bytes */
 809         l4_len = il2_hdr - l4.hdr;
 810         hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_S, l4_len >> 2);
 811
 812         /* define outer network header type */
 813         if (skb->protocol == htons(ETH_P_IP)) {
 814                 if (skb_is_gso(skb))
 815                         hns3_set_field(*ol_type_vlan_len_msec,
 816                                        HNS3_TXD_OL3T_S,
 817                                        HNS3_OL3T_IPV4_CSUM);
 818                 else
 819                         hns3_set_field(*ol_type_vlan_len_msec,
 820                                        HNS3_TXD_OL3T_S,
 821                                        HNS3_OL3T_IPV4_NO_CSUM);
 822
 823         } else if (skb->protocol == htons(ETH_P_IPV6)) {
 824                 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_S,
 825                                HNS3_OL3T_IPV6);
 826         }
 827
 828         if (ol4_proto == IPPROTO_UDP)
 829                 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
 830                                HNS3_TUN_MAC_IN_UDP);
 831         else if (ol4_proto == IPPROTO_GRE)
 832                 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
 833                                HNS3_TUN_NVGRE);
 834 }
 835
 836 static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
 837                            u8 il4_proto, u32 *type_cs_vlan_tso,
 838                            u32 *ol_type_vlan_len_msec)
 839 {
 840         unsigned char *l2_hdr = skb->data;
 841         u32 l4_proto = ol4_proto;
 842         union l4_hdr_info l4;
 843         union l3_hdr_info l3;
 844         u32 l2_len, l3_len;
 845
 846         l4.hdr = skb_transport_header(skb);
 847         l3.hdr = skb_network_header(skb);
 848
 849         /* handle encapsulation skb */
 850         if (skb->encapsulation) {
 851                 /* If this is a not UDP/GRE encapsulation skb */
 852                 if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) {
 853                         /* drop the skb tunnel packet if hardware don't support,
 854                          * because hardware can't calculate csum when TSO.
 855                          */
 856                         if (skb_is_gso(skb))
 857                                 return -EDOM;
 858
 859                         /* the stack computes the IP header already,
 860                          * driver calculate l4 checksum when not TSO.
 861                          */
 862                         skb_checksum_help(skb);
 863                         return 0;
 864                 }
 865
 866                 hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec);
 867
 868                 /* switch to inner header */
 869                 l2_hdr = skb_inner_mac_header(skb);
 870                 l3.hdr = skb_inner_network_header(skb);
 871                 l4.hdr = skb_inner_transport_header(skb);
 872                 l4_proto = il4_proto;
 873         }
 874
 875         if (l3.v4->version == 4) {
 876                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
 877                                HNS3_L3T_IPV4);
 878
 879                 /* the stack computes the IP header already, the only time we
 880                  * need the hardware to recompute it is in the case of TSO.
 881                  */
 882                 if (skb_is_gso(skb))
 883                         hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
 884         } else if (l3.v6->version == 6) {
 885                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
 886                                HNS3_L3T_IPV6);
 887         }
 888
 889         /* compute inner(/normal) L2 header size, defined in 2 Bytes */
 890         l2_len = l3.hdr - l2_hdr;
 891         hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_S, l2_len >> 1);
 892
 893         /* compute inner(/normal) L3 header size, defined in 4 Bytes */
 894         l3_len = l4.hdr - l3.hdr;
 895         hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_S, l3_len >> 2);
 896
 897         /* compute inner(/normal) L4 header size, defined in 4 Bytes */
 898         switch (l4_proto) {
 899         case IPPROTO_TCP:
 900                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 901                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
 902                                HNS3_L4T_TCP);
 903                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
 904                                l4.tcp->doff);
 905                 break;
 906         case IPPROTO_UDP:
 907                 if (hns3_tunnel_csum_bug(skb))
 908                         break;
 909
 910                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 911                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
 912                                HNS3_L4T_UDP);
 913                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
 914                                (sizeof(struct udphdr) >> 2));
 915                 break;
 916         case IPPROTO_SCTP:
 917                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 918                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
 919                                HNS3_L4T_SCTP);
 920                 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
 921                                (sizeof(struct sctphdr) >> 2));
 922                 break;
 923         default:
 924                 /* drop the skb tunnel packet if hardware don't support,
 925                  * because hardware can't calculate csum when TSO.
 926                  */
 927                 if (skb_is_gso(skb))
 928                         return -EDOM;
 929
 930                 /* the stack computes the IP header already,
 931                  * driver calculate l4 checksum when not TSO.
 932                  */
 933                 skb_checksum_help(skb);
 934                 return 0;
 935         }
 936
 937         return 0;
 938 }
 939
 940 static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring,
 941                              struct sk_buff *skb)
 942 {
 943         struct hnae3_handle *handle = tx_ring->tqp->handle;
 944         struct vlan_ethhdr *vhdr;
 945         int rc;
 946
 947         if (!(skb->protocol == htons(ETH_P_8021Q) ||
 948               skb_vlan_tag_present(skb)))
 949                 return 0;
 950
 951         /* Since HW limitation, if port based insert VLAN enabled, only one VLAN
 952          * header is allowed in skb, otherwise it will cause RAS error.
 953          */
 954         if (unlikely(skb_vlan_tagged_multi(skb) &&
 955                      handle->port_base_vlan_state ==
 956                      HNAE3_PORT_BASE_VLAN_ENABLE))
 957                 return -EINVAL;
 958
 959         if (skb->protocol == htons(ETH_P_8021Q) &&
 960             !(handle->kinfo.netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
 961                 /* When HW VLAN acceleration is turned off, and the stack
 962                  * sets the protocol to 802.1q, the driver just need to
 963                  * set the protocol to the encapsulated ethertype.
 964                  */
 965                 skb->protocol = vlan_get_protocol(skb);
 966                 return 0;
 967         }
 968
 969         if (skb_vlan_tag_present(skb)) {
 970                 /* Based on hw strategy, use out_vtag in two layer tag case,
 971                  * and use inner_vtag in one tag case.
 972                  */
 973                 if (skb->protocol == htons(ETH_P_8021Q) &&
 974                     handle->port_base_vlan_state ==
 975                     HNAE3_PORT_BASE_VLAN_DISABLE)
 976                         rc = HNS3_OUTER_VLAN_TAG;
 977                 else
 978                         rc = HNS3_INNER_VLAN_TAG;
 979
 980                 skb->protocol = vlan_get_protocol(skb);
 981                 return rc;
 982         }
 983
 984         rc = skb_cow_head(skb, 0);
 985         if (unlikely(rc < 0))
 986                 return rc;
 987
 988         vhdr = (struct vlan_ethhdr *)skb->data;
 989         vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority << VLAN_PRIO_SHIFT)
 990                                          & VLAN_PRIO_MASK);
 991
 992         skb->protocol = vlan_get_protocol(skb);
 993         return 0;
 994 }
 995
 996 static int hns3_fill_skb_desc(struct hns3_enet_ring *ring,
 997                               struct sk_buff *skb, struct hns3_desc *desc)
 998 {
 999         u32 ol_type_vlan_len_msec = 0;
1000         u32 type_cs_vlan_tso = 0;
1001         u32 paylen = skb->len;
1002         u16 inner_vtag = 0;
1003         u16 out_vtag = 0;
1004         u16 mss = 0;
1005         int ret;
1006
1007         ret = hns3_handle_vtags(ring, skb);
1008         if (unlikely(ret < 0)) {
1009                 u64_stats_update_begin(&ring->syncp);
1010                 ring->stats.tx_vlan_err++;
1011                 u64_stats_update_end(&ring->syncp);
1012                 return ret;
1013         } else if (ret == HNS3_INNER_VLAN_TAG) {
1014                 inner_vtag = skb_vlan_tag_get(skb);
1015                 inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1016                                 VLAN_PRIO_MASK;
1017                 hns3_set_field(type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1);
1018         } else if (ret == HNS3_OUTER_VLAN_TAG) {
1019                 out_vtag = skb_vlan_tag_get(skb);
1020                 out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1021                                 VLAN_PRIO_MASK;
1022                 hns3_set_field(ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B,
1023                                1);
1024         }
1025
1026         if (skb->ip_summed == CHECKSUM_PARTIAL) {
1027                 u8 ol4_proto, il4_proto;
1028
1029                 skb_reset_mac_len(skb);
1030
1031                 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1032                 if (unlikely(ret < 0)) {
1033                         u64_stats_update_begin(&ring->syncp);
1034                         ring->stats.tx_l4_proto_err++;
1035                         u64_stats_update_end(&ring->syncp);
1036                         return ret;
1037                 }
1038
1039                 ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto,
1040                                       &type_cs_vlan_tso,
1041                                       &ol_type_vlan_len_msec);
1042                 if (unlikely(ret < 0)) {
1043                         u64_stats_update_begin(&ring->syncp);
1044                         ring->stats.tx_l2l3l4_err++;
1045                         u64_stats_update_end(&ring->syncp);
1046                         return ret;
1047                 }
1048
1049                 ret = hns3_set_tso(skb, &paylen, &mss,
1050                                    &type_cs_vlan_tso);
1051                 if (unlikely(ret < 0)) {
1052                         u64_stats_update_begin(&ring->syncp);
1053                         ring->stats.tx_tso_err++;
1054                         u64_stats_update_end(&ring->syncp);
1055                         return ret;
1056                 }
1057         }
1058
1059         /* Set txbd */
1060         desc->tx.ol_type_vlan_len_msec =
1061                 cpu_to_le32(ol_type_vlan_len_msec);
1062         desc->tx.type_cs_vlan_tso_len = cpu_to_le32(type_cs_vlan_tso);
1063         desc->tx.paylen = cpu_to_le32(paylen);
1064         desc->tx.mss = cpu_to_le16(mss);
1065         desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1066         desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1067
1068         return 0;
1069 }
1070
1071 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
1072                           unsigned int size, enum hns_desc_type type)
1073 {
1074 #define HNS3_LIKELY_BD_NUM      1
1075
1076         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1077         struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1078         struct device *dev = ring_to_dev(ring);
1079         skb_frag_t *frag;
1080         unsigned int frag_buf_num;
1081         int k, sizeoflast;
1082         dma_addr_t dma;
1083
1084         if (type == DESC_TYPE_FRAGLIST_SKB ||
1085             type == DESC_TYPE_SKB) {
1086                 struct sk_buff *skb = (struct sk_buff *)priv;
1087
1088                 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1089         } else {
1090                 frag = (skb_frag_t *)priv;
1091                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1092         }
1093
1094         if (unlikely(dma_mapping_error(dev, dma))) {
1095                 u64_stats_update_begin(&ring->syncp);
1096                 ring->stats.sw_err_cnt++;
1097                 u64_stats_update_end(&ring->syncp);
1098                 return -ENOMEM;
1099         }
1100
1101         desc_cb->priv = priv;
1102         desc_cb->length = size;
1103         desc_cb->dma = dma;
1104         desc_cb->type = type;
1105
1106         if (likely(size <= HNS3_MAX_BD_SIZE)) {
1107                 desc->addr = cpu_to_le64(dma);
1108                 desc->tx.send_size = cpu_to_le16(size);
1109                 desc->tx.bdtp_fe_sc_vld_ra_ri =
1110                         cpu_to_le16(BIT(HNS3_TXD_VLD_B));
1111
1112                 trace_hns3_tx_desc(ring, ring->next_to_use);
1113                 ring_ptr_move_fw(ring, next_to_use);
1114                 return HNS3_LIKELY_BD_NUM;
1115         }
1116
1117         frag_buf_num = hns3_tx_bd_count(size);
1118         sizeoflast = size % HNS3_MAX_BD_SIZE;
1119         sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1120
1121         /* When frag size is bigger than hardware limit, split this frag */
1122         for (k = 0; k < frag_buf_num; k++) {
1123                 /* now, fill the descriptor */
1124                 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
1125                 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1126                                      (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1127                 desc->tx.bdtp_fe_sc_vld_ra_ri =
1128                                 cpu_to_le16(BIT(HNS3_TXD_VLD_B));
1129
1130                 trace_hns3_tx_desc(ring, ring->next_to_use);
1131                 /* move ring pointer to next */
1132                 ring_ptr_move_fw(ring, next_to_use);
1133
1134                 desc = &ring->desc[ring->next_to_use];
1135         }
1136
1137         return frag_buf_num;
1138 }
1139
1140 static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size,
1141                                     unsigned int bd_num)
1142 {
1143         unsigned int size;
1144         int i;
1145
1146         size = skb_headlen(skb);
1147         while (size > HNS3_MAX_BD_SIZE) {
1148                 bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1149                 size -= HNS3_MAX_BD_SIZE;
1150
1151                 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1152                         return bd_num;
1153         }
1154
1155         if (size) {
1156                 bd_size[bd_num++] = size;
1157                 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1158                         return bd_num;
1159         }
1160
1161         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1162                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1163                 size = skb_frag_size(frag);
1164                 if (!size)
1165                         continue;
1166
1167                 while (size > HNS3_MAX_BD_SIZE) {
1168                         bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1169                         size -= HNS3_MAX_BD_SIZE;
1170
1171                         if (bd_num > HNS3_MAX_TSO_BD_NUM)
1172                                 return bd_num;
1173                 }
1174
1175                 bd_size[bd_num++] = size;
1176                 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1177                         return bd_num;
1178         }
1179
1180         return bd_num;
1181 }
1182
1183 static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size)
1184 {
1185         struct sk_buff *frag_skb;
1186         unsigned int bd_num = 0;
1187
1188         /* If the total len is within the max bd limit */
1189         if (likely(skb->len <= HNS3_MAX_BD_SIZE && !skb_has_frag_list(skb) &&
1190                    skb_shinfo(skb)->nr_frags < HNS3_MAX_NON_TSO_BD_NUM))
1191                 return skb_shinfo(skb)->nr_frags + 1U;
1192
1193         /* The below case will always be linearized, return
1194          * HNS3_MAX_BD_NUM_TSO + 1U to make sure it is linearized.
1195          */
1196         if (unlikely(skb->len > HNS3_MAX_TSO_SIZE ||
1197                      (!skb_is_gso(skb) && skb->len > HNS3_MAX_NON_TSO_SIZE)))
1198                 return HNS3_MAX_TSO_BD_NUM + 1U;
1199
1200         bd_num = hns3_skb_bd_num(skb, bd_size, bd_num);
1201
1202         if (!skb_has_frag_list(skb) || bd_num > HNS3_MAX_TSO_BD_NUM)
1203                 return bd_num;
1204
1205         skb_walk_frags(skb, frag_skb) {
1206                 bd_num = hns3_skb_bd_num(frag_skb, bd_size, bd_num);
1207                 if (bd_num > HNS3_MAX_TSO_BD_NUM)
1208                         return bd_num;
1209         }
1210
1211         return bd_num;
1212 }
1213
1214 static unsigned int hns3_gso_hdr_len(struct sk_buff *skb)
1215 {
1216         if (!skb->encapsulation)
1217                 return skb_transport_offset(skb) + tcp_hdrlen(skb);
1218
1219         return skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb);
1220 }
1221
1222 /* HW need every continuous 8 buffer data to be larger than MSS,
1223  * we simplify it by ensuring skb_headlen + the first continuous
1224  * 7 frags to to be larger than gso header len + mss, and the remaining
1225  * continuous 7 frags to be larger than MSS except the last 7 frags.
1226  */
1227 static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size,
1228                                      unsigned int bd_num)
1229 {
1230         unsigned int tot_len = 0;
1231         int i;
1232
1233         for (i = 0; i < HNS3_MAX_NON_TSO_BD_NUM - 1U; i++)
1234                 tot_len += bd_size[i];
1235
1236         /* ensure the first 8 frags is greater than mss + header */
1237         if (tot_len + bd_size[HNS3_MAX_NON_TSO_BD_NUM - 1U] <
1238             skb_shinfo(skb)->gso_size + hns3_gso_hdr_len(skb))
1239                 return true;
1240
1241         /* ensure every continuous 7 buffer is greater than mss
1242          * except the last one.
1243          */
1244         for (i = 0; i < bd_num - HNS3_MAX_NON_TSO_BD_NUM; i++) {
1245                 tot_len -= bd_size[i];
1246                 tot_len += bd_size[i + HNS3_MAX_NON_TSO_BD_NUM - 1U];
1247
1248                 if (tot_len < skb_shinfo(skb)->gso_size)
1249                         return true;
1250         }
1251
1252         return false;
1253 }
1254
1255 void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size)
1256 {
1257         int i = 0;
1258
1259         for (i = 0; i < MAX_SKB_FRAGS; i++)
1260                 size[i] = skb_frag_size(&shinfo->frags[i]);
1261 }
1262
1263 static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring,
1264                                   struct net_device *netdev,
1265                                   struct sk_buff *skb)
1266 {
1267         struct hns3_nic_priv *priv = netdev_priv(netdev);
1268         unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U];
1269         unsigned int bd_num;
1270
1271         bd_num = hns3_tx_bd_num(skb, bd_size);
1272         if (unlikely(bd_num > HNS3_MAX_NON_TSO_BD_NUM)) {
1273                 if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) &&
1274                     !hns3_skb_need_linearized(skb, bd_size, bd_num)) {
1275                         trace_hns3_over_8bd(skb);
1276                         goto out;
1277                 }
1278
1279                 if (__skb_linearize(skb))
1280                         return -ENOMEM;
1281
1282                 bd_num = hns3_tx_bd_count(skb->len);
1283                 if ((skb_is_gso(skb) && bd_num > HNS3_MAX_TSO_BD_NUM) ||
1284                     (!skb_is_gso(skb) &&
1285                      bd_num > HNS3_MAX_NON_TSO_BD_NUM)) {
1286                         trace_hns3_over_8bd(skb);
1287                         return -ENOMEM;
1288                 }
1289
1290                 u64_stats_update_begin(&ring->syncp);
1291                 ring->stats.tx_copy++;
1292                 u64_stats_update_end(&ring->syncp);
1293         }
1294
1295 out:
1296         if (likely(ring_space(ring) >= bd_num))
1297                 return bd_num;
1298
1299         netif_stop_subqueue(netdev, ring->queue_index);
1300         smp_mb(); /* Memory barrier before checking ring_space */
1301
1302         /* Start queue in case hns3_clean_tx_ring has just made room
1303          * available and has not seen the queue stopped state performed
1304          * by netif_stop_subqueue above.
1305          */
1306         if (ring_space(ring) >= bd_num && netif_carrier_ok(netdev) &&
1307             !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
1308                 netif_start_subqueue(netdev, ring->queue_index);
1309                 return bd_num;
1310         }
1311
1312         return -EBUSY;
1313 }
1314
1315 static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
1316 {
1317         struct device *dev = ring_to_dev(ring);
1318         unsigned int i;
1319
1320         for (i = 0; i < ring->desc_num; i++) {
1321                 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1322
1323                 memset(desc, 0, sizeof(*desc));
1324
1325                 /* check if this is where we started */
1326                 if (ring->next_to_use == next_to_use_orig)
1327                         break;
1328
1329                 /* rollback one */
1330                 ring_ptr_move_bw(ring, next_to_use);
1331
1332                 if (!ring->desc_cb[ring->next_to_use].dma)
1333                         continue;
1334
1335                 /* unmap the descriptor dma address */
1336                 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB ||
1337                     ring->desc_cb[ring->next_to_use].type ==
1338                     DESC_TYPE_FRAGLIST_SKB)
1339                         dma_unmap_single(dev,
1340                                          ring->desc_cb[ring->next_to_use].dma,
1341                                         ring->desc_cb[ring->next_to_use].length,
1342                                         DMA_TO_DEVICE);
1343                 else if (ring->desc_cb[ring->next_to_use].length)
1344                         dma_unmap_page(dev,
1345                                        ring->desc_cb[ring->next_to_use].dma,
1346                                        ring->desc_cb[ring->next_to_use].length,
1347                                        DMA_TO_DEVICE);
1348
1349                 ring->desc_cb[ring->next_to_use].length = 0;
1350                 ring->desc_cb[ring->next_to_use].dma = 0;
1351                 ring->desc_cb[ring->next_to_use].type = DESC_TYPE_UNKNOWN;
1352         }
1353 }
1354
1355 static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring,
1356                                  struct sk_buff *skb, enum hns_desc_type type)
1357 {
1358         unsigned int size = skb_headlen(skb);
1359         int i, ret, bd_num = 0;
1360
1361         if (size) {
1362                 ret = hns3_fill_desc(ring, skb, size, type);
1363                 if (unlikely(ret < 0))
1364                         return ret;
1365
1366                 bd_num += ret;
1367         }
1368
1369         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1370                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1371
1372                 size = skb_frag_size(frag);
1373                 if (!size)
1374                         continue;
1375
1376                 ret = hns3_fill_desc(ring, frag, size, DESC_TYPE_PAGE);
1377                 if (unlikely(ret < 0))
1378                         return ret;
1379
1380                 bd_num += ret;
1381         }
1382
1383         return bd_num;
1384 }
1385
1386 static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num,
1387                              bool doorbell)
1388 {
1389         ring->pending_buf += num;
1390
1391         if (!doorbell) {
1392                 u64_stats_update_begin(&ring->syncp);
1393                 ring->stats.tx_more++;
1394                 u64_stats_update_end(&ring->syncp);
1395                 return;
1396         }
1397
1398         if (!ring->pending_buf)
1399                 return;
1400
1401         wmb(); /* Commit all data before submit */
1402
1403         hnae3_queue_xmit(ring->tqp, ring->pending_buf);
1404         ring->pending_buf = 0;
1405         WRITE_ONCE(ring->last_to_use, ring->next_to_use);
1406 }
1407
1408 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
1409 {
1410         struct hns3_nic_priv *priv = netdev_priv(netdev);
1411         struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping];
1412         struct netdev_queue *dev_queue;
1413         int pre_ntu, next_to_use_head;
1414         struct sk_buff *frag_skb;
1415         int bd_num = 0;
1416         bool doorbell;
1417         int ret;
1418
1419         /* Hardware can only handle short frames above 32 bytes */
1420         if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) {
1421                 hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
1422                 return NETDEV_TX_OK;
1423         }
1424
1425         /* Prefetch the data used later */
1426         prefetch(skb->data);
1427
1428         ret = hns3_nic_maybe_stop_tx(ring, netdev, skb);
1429         if (unlikely(ret <= 0)) {
1430                 if (ret == -EBUSY) {
1431                         u64_stats_update_begin(&ring->syncp);
1432                         ring->stats.tx_busy++;
1433                         u64_stats_update_end(&ring->syncp);
1434                         hns3_tx_doorbell(ring, 0, true);
1435                         return NETDEV_TX_BUSY;
1436                 } else if (ret == -ENOMEM) {
1437                         u64_stats_update_begin(&ring->syncp);
1438                         ring->stats.sw_err_cnt++;
1439                         u64_stats_update_end(&ring->syncp);
1440                 }
1441
1442                 hns3_rl_err(netdev, "xmit error: %d!\n", ret);
1443                 goto out_err_tx_ok;
1444         }
1445
1446         next_to_use_head = ring->next_to_use;
1447
1448         ret = hns3_fill_skb_desc(ring, skb, &ring->desc[ring->next_to_use]);
1449         if (unlikely(ret < 0))
1450                 goto fill_err;
1451
1452         ret = hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB);
1453         if (unlikely(ret < 0))
1454                 goto fill_err;
1455
1456         bd_num += ret;
1457
1458         skb_walk_frags(skb, frag_skb) {
1459                 ret = hns3_fill_skb_to_desc(ring, frag_skb,
1460                                             DESC_TYPE_FRAGLIST_SKB);
1461                 if (unlikely(ret < 0))
1462                         goto fill_err;
1463
1464                 bd_num += ret;
1465         }
1466
1467         pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) :
1468                                         (ring->desc_num - 1);
1469         ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |=
1470                                 cpu_to_le16(BIT(HNS3_TXD_FE_B));
1471         trace_hns3_tx_desc(ring, pre_ntu);
1472
1473         /* Complete translate all packets */
1474         dev_queue = netdev_get_tx_queue(netdev, ring->queue_index);
1475         doorbell = __netdev_tx_sent_queue(dev_queue, skb->len,
1476                                           netdev_xmit_more());
1477         hns3_tx_doorbell(ring, bd_num, doorbell);
1478
1479         return NETDEV_TX_OK;
1480
1481 fill_err:
1482         hns3_clear_desc(ring, next_to_use_head);
1483
1484 out_err_tx_ok:
1485         dev_kfree_skb_any(skb);
1486         hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
1487         return NETDEV_TX_OK;
1488 }
1489
1490 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1491 {
1492         struct hnae3_handle *h = hns3_get_handle(netdev);
1493         struct sockaddr *mac_addr = p;
1494         int ret;
1495
1496         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1497                 return -EADDRNOTAVAIL;
1498
1499         if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1500                 netdev_info(netdev, "already using mac address %pM\n",
1501                             mac_addr->sa_data);
1502                 return 0;
1503         }
1504
1505         /* For VF device, if there is a perm_addr, then the user will not
1506          * be allowed to change the address.
1507          */
1508         if (!hns3_is_phys_func(h->pdev) &&
1509             !is_zero_ether_addr(netdev->perm_addr)) {
1510                 netdev_err(netdev, "has permanent MAC %pM, user MAC %pM not allow\n",
1511                            netdev->perm_addr, mac_addr->sa_data);
1512                 return -EPERM;
1513         }
1514
1515         ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
1516         if (ret) {
1517                 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1518                 return ret;
1519         }
1520
1521         ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1522
1523         return 0;
1524 }
1525
1526 static int hns3_nic_do_ioctl(struct net_device *netdev,
1527                              struct ifreq *ifr, int cmd)
1528 {
1529         struct hnae3_handle *h = hns3_get_handle(netdev);
1530
1531         if (!netif_running(netdev))
1532                 return -EINVAL;
1533
1534         if (!h->ae_algo->ops->do_ioctl)
1535                 return -EOPNOTSUPP;
1536
1537         return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1538 }
1539
1540 static int hns3_nic_set_features(struct net_device *netdev,
1541                                  netdev_features_t features)
1542 {
1543         netdev_features_t changed = netdev->features ^ features;
1544         struct hns3_nic_priv *priv = netdev_priv(netdev);
1545         struct hnae3_handle *h = priv->ae_handle;
1546         bool enable;
1547         int ret;
1548
1549         if (changed & (NETIF_F_GRO_HW) && h->ae_algo->ops->set_gro_en) {
1550                 enable = !!(features & NETIF_F_GRO_HW);
1551                 ret = h->ae_algo->ops->set_gro_en(h, enable);
1552                 if (ret)
1553                         return ret;
1554         }
1555
1556         if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1557             h->ae_algo->ops->enable_hw_strip_rxvtag) {
1558                 enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
1559                 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, enable);
1560                 if (ret)
1561                         return ret;
1562         }
1563
1564         if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1565                 enable = !!(features & NETIF_F_NTUPLE);
1566                 h->ae_algo->ops->enable_fd(h, enable);
1567         }
1568
1569         netdev->features = features;
1570         return 0;
1571 }
1572
1573 static netdev_features_t hns3_features_check(struct sk_buff *skb,
1574                                              struct net_device *dev,
1575                                              netdev_features_t features)
1576 {
1577 #define HNS3_MAX_HDR_LEN        480U
1578 #define HNS3_MAX_L4_HDR_LEN     60U
1579
1580         size_t len;
1581
1582         if (skb->ip_summed != CHECKSUM_PARTIAL)
1583                 return features;
1584
1585         if (skb->encapsulation)
1586                 len = skb_inner_transport_header(skb) - skb->data;
1587         else
1588                 len = skb_transport_header(skb) - skb->data;
1589
1590         /* Assume L4 is 60 byte as TCP is the only protocol with a
1591          * a flexible value, and it's max len is 60 bytes.
1592          */
1593         len += HNS3_MAX_L4_HDR_LEN;
1594
1595         /* Hardware only supports checksum on the skb with a max header
1596          * len of 480 bytes.
1597          */
1598         if (len > HNS3_MAX_HDR_LEN)
1599                 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
1600
1601         return features;
1602 }
1603
1604 static void hns3_nic_get_stats64(struct net_device *netdev,
1605                                  struct rtnl_link_stats64 *stats)
1606 {
1607         struct hns3_nic_priv *priv = netdev_priv(netdev);
1608         int queue_num = priv->ae_handle->kinfo.num_tqps;
1609         struct hnae3_handle *handle = priv->ae_handle;
1610         struct hns3_enet_ring *ring;
1611         u64 rx_length_errors = 0;
1612         u64 rx_crc_errors = 0;
1613         u64 rx_multicast = 0;
1614         unsigned int start;
1615         u64 tx_errors = 0;
1616         u64 rx_errors = 0;
1617         unsigned int idx;
1618         u64 tx_bytes = 0;
1619         u64 rx_bytes = 0;
1620         u64 tx_pkts = 0;
1621         u64 rx_pkts = 0;
1622         u64 tx_drop = 0;
1623         u64 rx_drop = 0;
1624
1625         if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1626                 return;
1627
1628         handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1629
1630         for (idx = 0; idx < queue_num; idx++) {
1631                 /* fetch the tx stats */
1632                 ring = &priv->ring[idx];
1633                 do {
1634                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1635                         tx_bytes += ring->stats.tx_bytes;
1636                         tx_pkts += ring->stats.tx_pkts;
1637                         tx_drop += ring->stats.sw_err_cnt;
1638                         tx_drop += ring->stats.tx_vlan_err;
1639                         tx_drop += ring->stats.tx_l4_proto_err;
1640                         tx_drop += ring->stats.tx_l2l3l4_err;
1641                         tx_drop += ring->stats.tx_tso_err;
1642                         tx_errors += ring->stats.sw_err_cnt;
1643                         tx_errors += ring->stats.tx_vlan_err;
1644                         tx_errors += ring->stats.tx_l4_proto_err;
1645                         tx_errors += ring->stats.tx_l2l3l4_err;
1646                         tx_errors += ring->stats.tx_tso_err;
1647                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1648
1649                 /* fetch the rx stats */
1650                 ring = &priv->ring[idx + queue_num];
1651                 do {
1652                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1653                         rx_bytes += ring->stats.rx_bytes;
1654                         rx_pkts += ring->stats.rx_pkts;
1655                         rx_drop += ring->stats.l2_err;
1656                         rx_errors += ring->stats.l2_err;
1657                         rx_errors += ring->stats.l3l4_csum_err;
1658                         rx_crc_errors += ring->stats.l2_err;
1659                         rx_multicast += ring->stats.rx_multicast;
1660                         rx_length_errors += ring->stats.err_pkt_len;
1661                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1662         }
1663
1664         stats->tx_bytes = tx_bytes;
1665         stats->tx_packets = tx_pkts;
1666         stats->rx_bytes = rx_bytes;
1667         stats->rx_packets = rx_pkts;
1668
1669         stats->rx_errors = rx_errors;
1670         stats->multicast = rx_multicast;
1671         stats->rx_length_errors = rx_length_errors;
1672         stats->rx_crc_errors = rx_crc_errors;
1673         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1674
1675         stats->tx_errors = tx_errors;
1676         stats->rx_dropped = rx_drop;
1677         stats->tx_dropped = tx_drop;
1678         stats->collisions = netdev->stats.collisions;
1679         stats->rx_over_errors = netdev->stats.rx_over_errors;
1680         stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1681         stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1682         stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1683         stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1684         stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1685         stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1686         stats->tx_window_errors = netdev->stats.tx_window_errors;
1687         stats->rx_compressed = netdev->stats.rx_compressed;
1688         stats->tx_compressed = netdev->stats.tx_compressed;
1689 }
1690
1691 static int hns3_setup_tc(struct net_device *netdev, void *type_data)
1692 {
1693         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
1694         u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1695         struct hnae3_knic_private_info *kinfo;
1696         u8 tc = mqprio_qopt->qopt.num_tc;
1697         u16 mode = mqprio_qopt->mode;
1698         u8 hw = mqprio_qopt->qopt.hw;
1699         struct hnae3_handle *h;
1700
1701         if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1702                mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1703                 return -EOPNOTSUPP;
1704
1705         if (tc > HNAE3_MAX_TC)
1706                 return -EINVAL;
1707
1708         if (!netdev)
1709                 return -EINVAL;
1710
1711         h = hns3_get_handle(netdev);
1712         kinfo = &h->kinfo;
1713
1714         netif_dbg(h, drv, netdev, "setup tc: num_tc=%u\n", tc);
1715
1716         return (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1717                 kinfo->dcb_ops->setup_tc(h, tc ? tc : 1, prio_tc) : -EOPNOTSUPP;
1718 }
1719
1720 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1721                              void *type_data)
1722 {
1723         if (type != TC_SETUP_QDISC_MQPRIO)
1724                 return -EOPNOTSUPP;
1725
1726         return hns3_setup_tc(dev, type_data);
1727 }
1728
1729 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1730                                 __be16 proto, u16 vid)
1731 {
1732         struct hnae3_handle *h = hns3_get_handle(netdev);
1733         int ret = -EIO;
1734
1735         if (h->ae_algo->ops->set_vlan_filter)
1736                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1737
1738         return ret;
1739 }
1740
1741 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1742                                  __be16 proto, u16 vid)
1743 {
1744         struct hnae3_handle *h = hns3_get_handle(netdev);
1745         int ret = -EIO;
1746
1747         if (h->ae_algo->ops->set_vlan_filter)
1748                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1749
1750         return ret;
1751 }
1752
1753 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1754                                 u8 qos, __be16 vlan_proto)
1755 {
1756         struct hnae3_handle *h = hns3_get_handle(netdev);
1757         int ret = -EIO;
1758
1759         netif_dbg(h, drv, netdev,
1760                   "set vf vlan: vf=%d, vlan=%u, qos=%u, vlan_proto=0x%x\n",
1761                   vf, vlan, qos, ntohs(vlan_proto));
1762
1763         if (h->ae_algo->ops->set_vf_vlan_filter)
1764                 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1765                                                           qos, vlan_proto);
1766
1767         return ret;
1768 }
1769
1770 static int hns3_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)
1771 {
1772         struct hnae3_handle *handle = hns3_get_handle(netdev);
1773
1774         if (hns3_nic_resetting(netdev))
1775                 return -EBUSY;
1776
1777         if (!handle->ae_algo->ops->set_vf_spoofchk)
1778                 return -EOPNOTSUPP;
1779
1780         return handle->ae_algo->ops->set_vf_spoofchk(handle, vf, enable);
1781 }
1782
1783 static int hns3_set_vf_trust(struct net_device *netdev, int vf, bool enable)
1784 {
1785         struct hnae3_handle *handle = hns3_get_handle(netdev);
1786
1787         if (!handle->ae_algo->ops->set_vf_trust)
1788                 return -EOPNOTSUPP;
1789
1790         return handle->ae_algo->ops->set_vf_trust(handle, vf, enable);
1791 }
1792
1793 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1794 {
1795         struct hnae3_handle *h = hns3_get_handle(netdev);
1796         int ret;
1797
1798         if (hns3_nic_resetting(netdev))
1799                 return -EBUSY;
1800
1801         if (!h->ae_algo->ops->set_mtu)
1802                 return -EOPNOTSUPP;
1803
1804         netif_dbg(h, drv, netdev,
1805                   "change mtu from %u to %d\n", netdev->mtu, new_mtu);
1806
1807         ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1808         if (ret)
1809                 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1810                            ret);
1811         else
1812                 netdev->mtu = new_mtu;
1813
1814         return ret;
1815 }
1816
1817 static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1818 {
1819         struct hns3_nic_priv *priv = netdev_priv(ndev);
1820         struct hnae3_handle *h = hns3_get_handle(ndev);
1821         struct hns3_enet_ring *tx_ring;
1822         struct napi_struct *napi;
1823         int timeout_queue = 0;
1824         int hw_head, hw_tail;
1825         int fbd_num, fbd_oft;
1826         int ebd_num, ebd_oft;
1827         int bd_num, bd_err;
1828         int ring_en, tc;
1829         int i;
1830
1831         /* Find the stopped queue the same way the stack does */
1832         for (i = 0; i < ndev->num_tx_queues; i++) {
1833                 struct netdev_queue *q;
1834                 unsigned long trans_start;
1835
1836                 q = netdev_get_tx_queue(ndev, i);
1837                 trans_start = q->trans_start;
1838                 if (netif_xmit_stopped(q) &&
1839                     time_after(jiffies,
1840                                (trans_start + ndev->watchdog_timeo))) {
1841                         timeout_queue = i;
1842                         netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n",
1843                                     q->state,
1844                                     jiffies_to_msecs(jiffies - trans_start));
1845                         break;
1846                 }
1847         }
1848
1849         if (i == ndev->num_tx_queues) {
1850                 netdev_info(ndev,
1851                             "no netdev TX timeout queue found, timeout count: %llu\n",
1852                             priv->tx_timeout_count);
1853                 return false;
1854         }
1855
1856         priv->tx_timeout_count++;
1857
1858         tx_ring = &priv->ring[timeout_queue];
1859         napi = &tx_ring->tqp_vector->napi;
1860
1861         netdev_info(ndev,
1862                     "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, napi state: %lu\n",
1863                     priv->tx_timeout_count, timeout_queue, tx_ring->next_to_use,
1864                     tx_ring->next_to_clean, napi->state);
1865
1866         netdev_info(ndev,
1867                     "tx_pkts: %llu, tx_bytes: %llu, sw_err_cnt: %llu, tx_pending: %d\n",
1868                     tx_ring->stats.tx_pkts, tx_ring->stats.tx_bytes,
1869                     tx_ring->stats.sw_err_cnt, tx_ring->pending_buf);
1870
1871         netdev_info(ndev,
1872                     "seg_pkt_cnt: %llu, tx_more: %llu, restart_queue: %llu, tx_busy: %llu\n",
1873                     tx_ring->stats.seg_pkt_cnt, tx_ring->stats.tx_more,
1874                     tx_ring->stats.restart_queue, tx_ring->stats.tx_busy);
1875
1876         /* When mac received many pause frames continuous, it's unable to send
1877          * packets, which may cause tx timeout
1878          */
1879         if (h->ae_algo->ops->get_mac_stats) {
1880                 struct hns3_mac_stats mac_stats;
1881
1882                 h->ae_algo->ops->get_mac_stats(h, &mac_stats);
1883                 netdev_info(ndev, "tx_pause_cnt: %llu, rx_pause_cnt: %llu\n",
1884                             mac_stats.tx_pause_cnt, mac_stats.rx_pause_cnt);
1885         }
1886
1887         hw_head = readl_relaxed(tx_ring->tqp->io_base +
1888                                 HNS3_RING_TX_RING_HEAD_REG);
1889         hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1890                                 HNS3_RING_TX_RING_TAIL_REG);
1891         fbd_num = readl_relaxed(tx_ring->tqp->io_base +
1892                                 HNS3_RING_TX_RING_FBDNUM_REG);
1893         fbd_oft = readl_relaxed(tx_ring->tqp->io_base +
1894                                 HNS3_RING_TX_RING_OFFSET_REG);
1895         ebd_num = readl_relaxed(tx_ring->tqp->io_base +
1896                                 HNS3_RING_TX_RING_EBDNUM_REG);
1897         ebd_oft = readl_relaxed(tx_ring->tqp->io_base +
1898                                 HNS3_RING_TX_RING_EBD_OFFSET_REG);
1899         bd_num = readl_relaxed(tx_ring->tqp->io_base +
1900                                HNS3_RING_TX_RING_BD_NUM_REG);
1901         bd_err = readl_relaxed(tx_ring->tqp->io_base +
1902                                HNS3_RING_TX_RING_BD_ERR_REG);
1903         ring_en = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_EN_REG);
1904         tc = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_TX_RING_TC_REG);
1905
1906         netdev_info(ndev,
1907                     "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n",
1908                     bd_num, hw_head, hw_tail, bd_err,
1909                     readl(tx_ring->tqp_vector->mask_addr));
1910         netdev_info(ndev,
1911                     "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n",
1912                     ring_en, tc, fbd_num, fbd_oft, ebd_num, ebd_oft);
1913
1914         return true;
1915 }
1916
1917 static void hns3_nic_net_timeout(struct net_device *ndev, unsigned int txqueue)
1918 {
1919         struct hns3_nic_priv *priv = netdev_priv(ndev);
1920         struct hnae3_handle *h = priv->ae_handle;
1921
1922         if (!hns3_get_tx_timeo_queue_info(ndev))
1923                 return;
1924
1925         /* request the reset, and let the hclge to determine
1926          * which reset level should be done
1927          */
1928         if (h->ae_algo->ops->reset_event)
1929                 h->ae_algo->ops->reset_event(h->pdev, h);
1930 }
1931
1932 #ifdef CONFIG_RFS_ACCEL
1933 static int hns3_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
1934                               u16 rxq_index, u32 flow_id)
1935 {
1936         struct hnae3_handle *h = hns3_get_handle(dev);
1937         struct flow_keys fkeys;
1938
1939         if (!h->ae_algo->ops->add_arfs_entry)
1940                 return -EOPNOTSUPP;
1941
1942         if (skb->encapsulation)
1943                 return -EPROTONOSUPPORT;
1944
1945         if (!skb_flow_dissect_flow_keys(skb, &fkeys, 0))
1946                 return -EPROTONOSUPPORT;
1947
1948         if ((fkeys.basic.n_proto != htons(ETH_P_IP) &&
1949              fkeys.basic.n_proto != htons(ETH_P_IPV6)) ||
1950             (fkeys.basic.ip_proto != IPPROTO_TCP &&
1951              fkeys.basic.ip_proto != IPPROTO_UDP))
1952                 return -EPROTONOSUPPORT;
1953
1954         return h->ae_algo->ops->add_arfs_entry(h, rxq_index, flow_id, &fkeys);
1955 }
1956 #endif
1957
1958 static int hns3_nic_get_vf_config(struct net_device *ndev, int vf,
1959                                   struct ifla_vf_info *ivf)
1960 {
1961         struct hnae3_handle *h = hns3_get_handle(ndev);
1962
1963         if (!h->ae_algo->ops->get_vf_config)
1964                 return -EOPNOTSUPP;
1965
1966         return h->ae_algo->ops->get_vf_config(h, vf, ivf);
1967 }
1968
1969 static int hns3_nic_set_vf_link_state(struct net_device *ndev, int vf,
1970                                       int link_state)
1971 {
1972         struct hnae3_handle *h = hns3_get_handle(ndev);
1973
1974         if (!h->ae_algo->ops->set_vf_link_state)
1975                 return -EOPNOTSUPP;
1976
1977         return h->ae_algo->ops->set_vf_link_state(h, vf, link_state);
1978 }
1979
1980 static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf,
1981                                 int min_tx_rate, int max_tx_rate)
1982 {
1983         struct hnae3_handle *h = hns3_get_handle(ndev);
1984
1985         if (!h->ae_algo->ops->set_vf_rate)
1986                 return -EOPNOTSUPP;
1987
1988         return h->ae_algo->ops->set_vf_rate(h, vf, min_tx_rate, max_tx_rate,
1989                                             false);
1990 }
1991
1992 static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
1993 {
1994         struct hnae3_handle *h = hns3_get_handle(netdev);
1995
1996         if (!h->ae_algo->ops->set_vf_mac)
1997                 return -EOPNOTSUPP;
1998
1999         if (is_multicast_ether_addr(mac)) {
2000                 netdev_err(netdev,
2001                            "Invalid MAC:%pM specified. Could not set MAC\n",
2002                            mac);
2003                 return -EINVAL;
2004         }
2005
2006         return h->ae_algo->ops->set_vf_mac(h, vf_id, mac);
2007 }
2008
2009 static const struct net_device_ops hns3_nic_netdev_ops = {
2010         .ndo_open               = hns3_nic_net_open,
2011         .ndo_stop               = hns3_nic_net_stop,
2012         .ndo_start_xmit         = hns3_nic_net_xmit,
2013         .ndo_tx_timeout         = hns3_nic_net_timeout,
2014         .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
2015         .ndo_do_ioctl           = hns3_nic_do_ioctl,
2016         .ndo_change_mtu         = hns3_nic_change_mtu,
2017         .ndo_set_features       = hns3_nic_set_features,
2018         .ndo_features_check     = hns3_features_check,
2019         .ndo_get_stats64        = hns3_nic_get_stats64,
2020         .ndo_setup_tc           = hns3_nic_setup_tc,
2021         .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
2022         .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
2023         .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
2024         .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
2025         .ndo_set_vf_spoofchk    = hns3_set_vf_spoofchk,
2026         .ndo_set_vf_trust       = hns3_set_vf_trust,
2027 #ifdef CONFIG_RFS_ACCEL
2028         .ndo_rx_flow_steer      = hns3_rx_flow_steer,
2029 #endif
2030         .ndo_get_vf_config      = hns3_nic_get_vf_config,
2031         .ndo_set_vf_link_state  = hns3_nic_set_vf_link_state,
2032         .ndo_set_vf_rate        = hns3_nic_set_vf_rate,
2033         .ndo_set_vf_mac         = hns3_nic_set_vf_mac,
2034 };
2035
2036 bool hns3_is_phys_func(struct pci_dev *pdev)
2037 {
2038         u32 dev_id = pdev->device;
2039
2040         switch (dev_id) {
2041         case HNAE3_DEV_ID_GE:
2042         case HNAE3_DEV_ID_25GE:
2043         case HNAE3_DEV_ID_25GE_RDMA:
2044         case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
2045         case HNAE3_DEV_ID_50GE_RDMA:
2046         case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
2047         case HNAE3_DEV_ID_100G_RDMA_MACSEC:
2048                 return true;
2049         case HNAE3_DEV_ID_100G_VF:
2050         case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
2051                 return false;
2052         default:
2053                 dev_warn(&pdev->dev, "un-recognized pci device-id %u",
2054                          dev_id);
2055         }
2056
2057         return false;
2058 }
2059
2060 static void hns3_disable_sriov(struct pci_dev *pdev)
2061 {
2062         /* If our VFs are assigned we cannot shut down SR-IOV
2063          * without causing issues, so just leave the hardware
2064          * available but disabled
2065          */
2066         if (pci_vfs_assigned(pdev)) {
2067                 dev_warn(&pdev->dev,
2068                          "disabling driver while VFs are assigned\n");
2069                 return;
2070         }
2071
2072         pci_disable_sriov(pdev);
2073 }
2074
2075 static void hns3_get_dev_capability(struct pci_dev *pdev,
2076                                     struct hnae3_ae_dev *ae_dev)
2077 {
2078         if (pdev->revision >= 0x21) {
2079                 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_FD_B, 1);
2080                 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_GRO_B, 1);
2081         }
2082 }
2083
2084 /* hns3_probe - Device initialization routine
2085  * @pdev: PCI device information struct
2086  * @ent: entry in hns3_pci_tbl
2087  *
2088  * hns3_probe initializes a PF identified by a pci_dev structure.
2089  * The OS initialization, configuring of the PF private structure,
2090  * and a hardware reset occur.
2091  *
2092  * Returns 0 on success, negative on failure
2093  */
2094 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2095 {
2096         struct hnae3_ae_dev *ae_dev;
2097         int ret;
2098
2099         ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev), GFP_KERNEL);
2100         if (!ae_dev)
2101                 return -ENOMEM;
2102
2103         ae_dev->pdev = pdev;
2104         ae_dev->flag = ent->driver_data;
2105         hns3_get_dev_capability(pdev, ae_dev);
2106         pci_set_drvdata(pdev, ae_dev);
2107
2108         ret = hnae3_register_ae_dev(ae_dev);
2109         if (ret)
2110                 pci_set_drvdata(pdev, NULL);
2111
2112         return ret;
2113 }
2114
2115 /* hns3_remove - Device removal routine
2116  * @pdev: PCI device information struct
2117  */
2118 static void hns3_remove(struct pci_dev *pdev)
2119 {
2120         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2121
2122         if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
2123                 hns3_disable_sriov(pdev);
2124
2125         hnae3_unregister_ae_dev(ae_dev);
2126         pci_set_drvdata(pdev, NULL);
2127 }
2128
2129 /**
2130  * hns3_pci_sriov_configure
2131  * @pdev: pointer to a pci_dev structure
2132  * @num_vfs: number of VFs to allocate
2133  *
2134  * Enable or change the number of VFs. Called when the user updates the number
2135  * of VFs in sysfs.
2136  **/
2137 static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
2138 {
2139         int ret;
2140
2141         if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
2142                 dev_warn(&pdev->dev, "Can not config SRIOV\n");
2143                 return -EINVAL;
2144         }
2145
2146         if (num_vfs) {
2147                 ret = pci_enable_sriov(pdev, num_vfs);
2148                 if (ret)
2149                         dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
2150                 else
2151                         return num_vfs;
2152         } else if (!pci_vfs_assigned(pdev)) {
2153                 pci_disable_sriov(pdev);
2154         } else {
2155                 dev_warn(&pdev->dev,
2156                          "Unable to free VFs because some are assigned to VMs.\n");
2157         }
2158
2159         return 0;
2160 }
2161
2162 static void hns3_shutdown(struct pci_dev *pdev)
2163 {
2164         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2165
2166         hnae3_unregister_ae_dev(ae_dev);
2167         pci_set_drvdata(pdev, NULL);
2168
2169         if (system_state == SYSTEM_POWER_OFF)
2170                 pci_set_power_state(pdev, PCI_D3hot);
2171 }
2172
2173 static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
2174                                             pci_channel_state_t state)
2175 {
2176         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2177         pci_ers_result_t ret;
2178
2179         dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
2180
2181         if (state == pci_channel_io_perm_failure)
2182                 return PCI_ERS_RESULT_DISCONNECT;
2183
2184         if (!ae_dev || !ae_dev->ops) {
2185                 dev_err(&pdev->dev,
2186                         "Can't recover - error happened before device initialized\n");
2187                 return PCI_ERS_RESULT_NONE;
2188         }
2189
2190         if (ae_dev->ops->handle_hw_ras_error)
2191                 ret = ae_dev->ops->handle_hw_ras_error(ae_dev);
2192         else
2193                 return PCI_ERS_RESULT_NONE;
2194
2195         return ret;
2196 }
2197
2198 static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
2199 {
2200         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2201         const struct hnae3_ae_ops *ops;
2202         enum hnae3_reset_type reset_type;
2203         struct device *dev = &pdev->dev;
2204
2205         if (!ae_dev || !ae_dev->ops)
2206                 return PCI_ERS_RESULT_NONE;
2207
2208         ops = ae_dev->ops;
2209         /* request the reset */
2210         if (ops->reset_event && ops->get_reset_level &&
2211             ops->set_default_reset_request) {
2212                 if (ae_dev->hw_err_reset_req) {
2213                         reset_type = ops->get_reset_level(ae_dev,
2214                                                 &ae_dev->hw_err_reset_req);
2215                         ops->set_default_reset_request(ae_dev, reset_type);
2216                         dev_info(dev, "requesting reset due to PCI error\n");
2217                         ops->reset_event(pdev, NULL);
2218                 }
2219
2220                 return PCI_ERS_RESULT_RECOVERED;
2221         }
2222
2223         return PCI_ERS_RESULT_DISCONNECT;
2224 }
2225
2226 static void hns3_reset_prepare(struct pci_dev *pdev)
2227 {
2228         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2229
2230         dev_info(&pdev->dev, "FLR prepare\n");
2231         if (ae_dev && ae_dev->ops && ae_dev->ops->flr_prepare)
2232                 ae_dev->ops->flr_prepare(ae_dev);
2233 }
2234
2235 static void hns3_reset_done(struct pci_dev *pdev)
2236 {
2237         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2238
2239         dev_info(&pdev->dev, "FLR done\n");
2240         if (ae_dev && ae_dev->ops && ae_dev->ops->flr_done)
2241                 ae_dev->ops->flr_done(ae_dev);
2242 }
2243
2244 static const struct pci_error_handlers hns3_err_handler = {
2245         .error_detected = hns3_error_detected,
2246         .slot_reset     = hns3_slot_reset,
2247         .reset_prepare  = hns3_reset_prepare,
2248         .reset_done     = hns3_reset_done,
2249 };
2250
2251 static struct pci_driver hns3_driver = {
2252         .name     = hns3_driver_name,
2253         .id_table = hns3_pci_tbl,
2254         .probe    = hns3_probe,
2255         .remove   = hns3_remove,
2256         .shutdown = hns3_shutdown,
2257         .sriov_configure = hns3_pci_sriov_configure,
2258         .err_handler    = &hns3_err_handler,
2259 };
2260
2261 /* set default feature to hns3 */
2262 static void hns3_set_default_feature(struct net_device *netdev)
2263 {
2264         struct hnae3_handle *h = hns3_get_handle(netdev);
2265         struct pci_dev *pdev = h->pdev;
2266
2267         netdev->priv_flags |= IFF_UNICAST_FLT;
2268
2269         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2270                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2271                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2272                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2273                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC |
2274                 NETIF_F_TSO_MANGLEID | NETIF_F_FRAGLIST;
2275
2276         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
2277
2278         netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2279                 NETIF_F_HW_VLAN_CTAG_FILTER |
2280                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2281                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2282                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2283                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2284                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC |
2285                 NETIF_F_FRAGLIST;
2286
2287         netdev->vlan_features |=
2288                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
2289                 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
2290                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2291                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2292                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC |
2293                 NETIF_F_FRAGLIST;
2294
2295         netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2296                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2297                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2298                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2299                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2300                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC |
2301                 NETIF_F_FRAGLIST;
2302
2303         if (pdev->revision >= 0x21) {
2304                 netdev->hw_features |= NETIF_F_GRO_HW;
2305                 netdev->features |= NETIF_F_GRO_HW;
2306
2307                 if (!(h->flags & HNAE3_SUPPORT_VF)) {
2308                         netdev->hw_features |= NETIF_F_NTUPLE;
2309                         netdev->features |= NETIF_F_NTUPLE;
2310                 }
2311         }
2312 }
2313
2314 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
2315                              struct hns3_desc_cb *cb)
2316 {
2317         unsigned int order = hns3_page_order(ring);
2318         struct page *p;
2319
2320         p = dev_alloc_pages(order);
2321         if (!p)
2322                 return -ENOMEM;
2323
2324         cb->priv = p;
2325         cb->page_offset = 0;
2326         cb->reuse_flag = 0;
2327         cb->buf  = page_address(p);
2328         cb->length = hns3_page_size(ring);
2329         cb->type = DESC_TYPE_PAGE;
2330         page_ref_add(p, USHRT_MAX - 1);
2331         cb->pagecnt_bias = USHRT_MAX;
2332
2333         return 0;
2334 }
2335
2336 static void hns3_free_buffer(struct hns3_enet_ring *ring,
2337                              struct hns3_desc_cb *cb)
2338 {
2339         if (cb->type == DESC_TYPE_SKB)
2340                 dev_kfree_skb_any((struct sk_buff *)cb->priv);
2341         else if (!HNAE3_IS_TX_RING(ring) && cb->pagecnt_bias)
2342                 __page_frag_cache_drain(cb->priv, cb->pagecnt_bias);
2343         memset(cb, 0, sizeof(*cb));
2344 }
2345
2346 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
2347 {
2348         cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
2349                                cb->length, ring_to_dma_dir(ring));
2350
2351         if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
2352                 return -EIO;
2353
2354         return 0;
2355 }
2356
2357 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
2358                               struct hns3_desc_cb *cb)
2359 {
2360         if (cb->type == DESC_TYPE_SKB || cb->type == DESC_TYPE_FRAGLIST_SKB)
2361                 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
2362                                  ring_to_dma_dir(ring));
2363         else if (cb->length)
2364                 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
2365                                ring_to_dma_dir(ring));
2366 }
2367
2368 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
2369 {
2370         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2371         ring->desc[i].addr = 0;
2372 }
2373
2374 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
2375 {
2376         struct hns3_desc_cb *cb = &ring->desc_cb[i];
2377
2378         if (!ring->desc_cb[i].dma)
2379                 return;
2380
2381         hns3_buffer_detach(ring, i);
2382         hns3_free_buffer(ring, cb);
2383 }
2384
2385 static void hns3_free_buffers(struct hns3_enet_ring *ring)
2386 {
2387         int i;
2388
2389         for (i = 0; i < ring->desc_num; i++)
2390                 hns3_free_buffer_detach(ring, i);
2391 }
2392
2393 /* free desc along with its attached buffer */
2394 static void hns3_free_desc(struct hns3_enet_ring *ring)
2395 {
2396         int size = ring->desc_num * sizeof(ring->desc[0]);
2397
2398         hns3_free_buffers(ring);
2399
2400         if (ring->desc) {
2401                 dma_free_coherent(ring_to_dev(ring), size,
2402                                   ring->desc, ring->desc_dma_addr);
2403                 ring->desc = NULL;
2404         }
2405 }
2406
2407 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
2408 {
2409         int size = ring->desc_num * sizeof(ring->desc[0]);
2410
2411         ring->desc = dma_alloc_coherent(ring_to_dev(ring), size,
2412                                         &ring->desc_dma_addr, GFP_KERNEL);
2413         if (!ring->desc)
2414                 return -ENOMEM;
2415
2416         return 0;
2417 }
2418
2419 static int hns3_alloc_and_map_buffer(struct hns3_enet_ring *ring,
2420                                    struct hns3_desc_cb *cb)
2421 {
2422         int ret;
2423
2424         ret = hns3_alloc_buffer(ring, cb);
2425         if (ret)
2426                 goto out;
2427
2428         ret = hns3_map_buffer(ring, cb);
2429         if (ret)
2430                 goto out_with_buf;
2431
2432         return 0;
2433
2434 out_with_buf:
2435         hns3_free_buffer(ring, cb);
2436 out:
2437         return ret;
2438 }
2439
2440 static int hns3_alloc_and_attach_buffer(struct hns3_enet_ring *ring, int i)
2441 {
2442         int ret = hns3_alloc_and_map_buffer(ring, &ring->desc_cb[i]);
2443
2444         if (ret)
2445                 return ret;
2446
2447         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2448
2449         return 0;
2450 }
2451
2452 /* Allocate memory for raw pkg, and map with dma */
2453 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2454 {
2455         int i, j, ret;
2456
2457         for (i = 0; i < ring->desc_num; i++) {
2458                 ret = hns3_alloc_and_attach_buffer(ring, i);
2459                 if (ret)
2460                         goto out_buffer_fail;
2461         }
2462
2463         return 0;
2464
2465 out_buffer_fail:
2466         for (j = i - 1; j >= 0; j--)
2467                 hns3_free_buffer_detach(ring, j);
2468         return ret;
2469 }
2470
2471 /* detach a in-used buffer and replace with a reserved one */
2472 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2473                                 struct hns3_desc_cb *res_cb)
2474 {
2475         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2476         ring->desc_cb[i] = *res_cb;
2477         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2478         ring->desc[i].rx.bd_base_info = 0;
2479 }
2480
2481 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2482 {
2483         ring->desc_cb[i].reuse_flag = 0;
2484         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +
2485                                          ring->desc_cb[i].page_offset);
2486         ring->desc[i].rx.bd_base_info = 0;
2487
2488         dma_sync_single_for_device(ring_to_dev(ring),
2489                         ring->desc_cb[i].dma + ring->desc_cb[i].page_offset,
2490                         hns3_buf_size(ring),
2491                         DMA_FROM_DEVICE);
2492 }
2493
2494 static bool hns3_nic_reclaim_desc(struct hns3_enet_ring *ring,
2495                                   int *bytes, int *pkts)
2496 {
2497         /* pair with ring->last_to_use update in hns3_tx_doorbell(),
2498          * smp_store_release() is not used in hns3_tx_doorbell() because
2499          * the doorbell operation already have the needed barrier operation.
2500          */
2501         int ltu = smp_load_acquire(&ring->last_to_use);
2502         int ntc = ring->next_to_clean;
2503         struct hns3_desc_cb *desc_cb;
2504         bool reclaimed = false;
2505         struct hns3_desc *desc;
2506
2507         while (ltu != ntc) {
2508                 desc = &ring->desc[ntc];
2509
2510                 if (le16_to_cpu(desc->tx.bdtp_fe_sc_vld_ra_ri) &
2511                                 BIT(HNS3_TXD_VLD_B))
2512                         break;
2513
2514                 desc_cb = &ring->desc_cb[ntc];
2515                 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
2516                 (*bytes) += desc_cb->length;
2517                 /* desc_cb will be cleaned, after hnae3_free_buffer_detach */
2518                 hns3_free_buffer_detach(ring, ntc);
2519
2520                 if (++ntc == ring->desc_num)
2521                         ntc = 0;
2522
2523                 /* Issue prefetch for next Tx descriptor */
2524                 prefetch(&ring->desc_cb[ntc]);
2525                 reclaimed = true;
2526         }
2527
2528         if (unlikely(!reclaimed))
2529                 return false;
2530
2531         /* This smp_store_release() pairs with smp_load_acquire() in
2532          * ring_space called by hns3_nic_net_xmit.
2533          */
2534         smp_store_release(&ring->next_to_clean, ntc);
2535         return true;
2536 }
2537
2538 void hns3_clean_tx_ring(struct hns3_enet_ring *ring)
2539 {
2540         struct net_device *netdev = ring_to_netdev(ring);
2541         struct hns3_nic_priv *priv = netdev_priv(netdev);
2542         struct netdev_queue *dev_queue;
2543         int bytes, pkts;
2544
2545         bytes = 0;
2546         pkts = 0;
2547
2548         if (unlikely(!hns3_nic_reclaim_desc(ring, &bytes, &pkts)))
2549                 return;
2550
2551         ring->tqp_vector->tx_group.total_bytes += bytes;
2552         ring->tqp_vector->tx_group.total_packets += pkts;
2553
2554         u64_stats_update_begin(&ring->syncp);
2555         ring->stats.tx_bytes += bytes;
2556         ring->stats.tx_pkts += pkts;
2557         u64_stats_update_end(&ring->syncp);
2558
2559         dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2560         netdev_tx_completed_queue(dev_queue, pkts, bytes);
2561
2562         if (unlikely(netif_carrier_ok(netdev) &&
2563                      ring_space(ring) > HNS3_MAX_TSO_BD_NUM)) {
2564                 /* Make sure that anybody stopping the queue after this
2565                  * sees the new next_to_clean.
2566                  */
2567                 smp_mb();
2568                 if (netif_tx_queue_stopped(dev_queue) &&
2569                     !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
2570                         netif_tx_wake_queue(dev_queue);
2571                         ring->stats.restart_queue++;
2572                 }
2573         }
2574 }
2575
2576 static int hns3_desc_unused(struct hns3_enet_ring *ring)
2577 {
2578         int ntc = ring->next_to_clean;
2579         int ntu = ring->next_to_use;
2580
2581         return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2582 }
2583
2584 static void hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring,
2585                                       int cleand_count)
2586 {
2587         struct hns3_desc_cb *desc_cb;
2588         struct hns3_desc_cb res_cbs;
2589         int i, ret;
2590
2591         for (i = 0; i < cleand_count; i++) {
2592                 desc_cb = &ring->desc_cb[ring->next_to_use];
2593                 if (desc_cb->reuse_flag) {
2594                         u64_stats_update_begin(&ring->syncp);
2595                         ring->stats.reuse_pg_cnt++;
2596                         u64_stats_update_end(&ring->syncp);
2597
2598                         hns3_reuse_buffer(ring, ring->next_to_use);
2599                 } else {
2600                         ret = hns3_alloc_and_map_buffer(ring, &res_cbs);
2601                         if (ret) {
2602                                 u64_stats_update_begin(&ring->syncp);
2603                                 ring->stats.sw_err_cnt++;
2604                                 u64_stats_update_end(&ring->syncp);
2605
2606                                 hns3_rl_err(ring_to_netdev(ring),
2607                                             "alloc rx buffer failed: %d\n",
2608                                             ret);
2609                                 break;
2610                         }
2611                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2612
2613                         u64_stats_update_begin(&ring->syncp);
2614                         ring->stats.non_reuse_pg++;
2615                         u64_stats_update_end(&ring->syncp);
2616                 }
2617
2618                 ring_ptr_move_fw(ring, next_to_use);
2619         }
2620
2621         wmb(); /* Make all data has been write before submit */
2622         writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2623 }
2624
2625 static bool hns3_page_is_reusable(struct page *page)
2626 {
2627         return page_to_nid(page) == numa_mem_id() &&
2628                 !page_is_pfmemalloc(page);
2629 }
2630
2631 static bool hns3_can_reuse_page(struct hns3_desc_cb *cb)
2632 {
2633         return (page_count(cb->priv) - cb->pagecnt_bias) == 1;
2634 }
2635
2636 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2637                                 struct hns3_enet_ring *ring, int pull_len,
2638                                 struct hns3_desc_cb *desc_cb)
2639 {
2640         struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2641         int size = le16_to_cpu(desc->rx.size);
2642         u32 truesize = hns3_buf_size(ring);
2643
2644         desc_cb->pagecnt_bias--;
2645         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2646                         size - pull_len, truesize);
2647
2648         /* Avoid re-using remote pages, or the stack is still using the page
2649          * when page_offset rollback to zero, flag default unreuse
2650          */
2651         if (unlikely(!hns3_page_is_reusable(desc_cb->priv)) ||
2652             (!desc_cb->page_offset && !hns3_can_reuse_page(desc_cb))) {
2653                 __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias);
2654                 return;
2655         }
2656
2657         /* Move offset up to the next cache line */
2658         desc_cb->page_offset += truesize;
2659
2660         if (desc_cb->page_offset + truesize <= hns3_page_size(ring)) {
2661                 desc_cb->reuse_flag = 1;
2662         } else if (hns3_can_reuse_page(desc_cb)) {
2663                 desc_cb->reuse_flag = 1;
2664                 desc_cb->page_offset = 0;
2665         } else if (desc_cb->pagecnt_bias) {
2666                 __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias);
2667                 return;
2668         }
2669
2670         if (unlikely(!desc_cb->pagecnt_bias)) {
2671                 page_ref_add(desc_cb->priv, USHRT_MAX);
2672                 desc_cb->pagecnt_bias = USHRT_MAX;
2673         }
2674 }
2675
2676 static int hns3_gro_complete(struct sk_buff *skb, u32 l234info)
2677 {
2678         __be16 type = skb->protocol;
2679         struct tcphdr *th;
2680         int depth = 0;
2681
2682         while (eth_type_vlan(type)) {
2683                 struct vlan_hdr *vh;
2684
2685                 if ((depth + VLAN_HLEN) > skb_headlen(skb))
2686                         return -EFAULT;
2687
2688                 vh = (struct vlan_hdr *)(skb->data + depth);
2689                 type = vh->h_vlan_encapsulated_proto;
2690                 depth += VLAN_HLEN;
2691         }
2692
2693         skb_set_network_header(skb, depth);
2694
2695         if (type == htons(ETH_P_IP)) {
2696                 const struct iphdr *iph = ip_hdr(skb);
2697
2698                 depth += sizeof(struct iphdr);
2699                 skb_set_transport_header(skb, depth);
2700                 th = tcp_hdr(skb);
2701                 th->check = ~tcp_v4_check(skb->len - depth, iph->saddr,
2702                                           iph->daddr, 0);
2703         } else if (type == htons(ETH_P_IPV6)) {
2704                 const struct ipv6hdr *iph = ipv6_hdr(skb);
2705
2706                 depth += sizeof(struct ipv6hdr);
2707                 skb_set_transport_header(skb, depth);
2708                 th = tcp_hdr(skb);
2709                 th->check = ~tcp_v6_check(skb->len - depth, &iph->saddr,
2710                                           &iph->daddr, 0);
2711         } else {
2712                 hns3_rl_err(skb->dev,
2713                             "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n",
2714                             be16_to_cpu(type), depth);
2715                 return -EFAULT;
2716         }
2717
2718         skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
2719         if (th->cwr)
2720                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
2721
2722         if (l234info & BIT(HNS3_RXD_GRO_FIXID_B))
2723                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID;
2724
2725         skb->csum_start = (unsigned char *)th - skb->head;
2726         skb->csum_offset = offsetof(struct tcphdr, check);
2727         skb->ip_summed = CHECKSUM_PARTIAL;
2728
2729         trace_hns3_gro(skb);
2730
2731         return 0;
2732 }
2733
2734 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2735                              u32 l234info, u32 bd_base_info, u32 ol_info)
2736 {
2737         struct net_device *netdev = ring_to_netdev(ring);
2738         int l3_type, l4_type;
2739         int ol4_type;
2740
2741         skb->ip_summed = CHECKSUM_NONE;
2742
2743         skb_checksum_none_assert(skb);
2744
2745         if (!(netdev->features & NETIF_F_RXCSUM))
2746                 return;
2747
2748         /* check if hardware has done checksum */
2749         if (!(bd_base_info & BIT(HNS3_RXD_L3L4P_B)))
2750                 return;
2751
2752         if (unlikely(l234info & (BIT(HNS3_RXD_L3E_B) | BIT(HNS3_RXD_L4E_B) |
2753                                  BIT(HNS3_RXD_OL3E_B) |
2754                                  BIT(HNS3_RXD_OL4E_B)))) {
2755                 u64_stats_update_begin(&ring->syncp);
2756                 ring->stats.l3l4_csum_err++;
2757                 u64_stats_update_end(&ring->syncp);
2758
2759                 return;
2760         }
2761
2762         ol4_type = hnae3_get_field(ol_info, HNS3_RXD_OL4ID_M,
2763                                    HNS3_RXD_OL4ID_S);
2764         switch (ol4_type) {
2765         case HNS3_OL4_TYPE_MAC_IN_UDP:
2766         case HNS3_OL4_TYPE_NVGRE:
2767                 skb->csum_level = 1;
2768                 fallthrough;
2769         case HNS3_OL4_TYPE_NO_TUN:
2770                 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2771                                           HNS3_RXD_L3ID_S);
2772                 l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2773                                           HNS3_RXD_L4ID_S);
2774
2775                 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2776                 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2777                      l3_type == HNS3_L3_TYPE_IPV6) &&
2778                     (l4_type == HNS3_L4_TYPE_UDP ||
2779                      l4_type == HNS3_L4_TYPE_TCP ||
2780                      l4_type == HNS3_L4_TYPE_SCTP))
2781                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2782                 break;
2783         default:
2784                 break;
2785         }
2786 }
2787
2788 static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2789 {
2790         if (skb_has_frag_list(skb))
2791                 napi_gro_flush(&ring->tqp_vector->napi, false);
2792
2793         napi_gro_receive(&ring->tqp_vector->napi, skb);
2794 }
2795
2796 static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2797                                 struct hns3_desc *desc, u32 l234info,
2798                                 u16 *vlan_tag)
2799 {
2800         struct hnae3_handle *handle = ring->tqp->handle;
2801         struct pci_dev *pdev = ring->tqp->handle->pdev;
2802
2803         if (pdev->revision == 0x20) {
2804                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2805                 if (!(*vlan_tag & VLAN_VID_MASK))
2806                         *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2807
2808                 return (*vlan_tag != 0);
2809         }
2810
2811 #define HNS3_STRP_OUTER_VLAN    0x1
2812 #define HNS3_STRP_INNER_VLAN    0x2
2813 #define HNS3_STRP_BOTH          0x3
2814
2815         /* Hardware always insert VLAN tag into RX descriptor when
2816          * remove the tag from packet, driver needs to determine
2817          * reporting which tag to stack.
2818          */
2819         switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2820                                 HNS3_RXD_STRP_TAGP_S)) {
2821         case HNS3_STRP_OUTER_VLAN:
2822                 if (handle->port_base_vlan_state !=
2823                                 HNAE3_PORT_BASE_VLAN_DISABLE)
2824                         return false;
2825
2826                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2827                 return true;
2828         case HNS3_STRP_INNER_VLAN:
2829                 if (handle->port_base_vlan_state !=
2830                                 HNAE3_PORT_BASE_VLAN_DISABLE)
2831                         return false;
2832
2833                 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2834                 return true;
2835         case HNS3_STRP_BOTH:
2836                 if (handle->port_base_vlan_state ==
2837                                 HNAE3_PORT_BASE_VLAN_DISABLE)
2838                         *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2839                 else
2840                         *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2841
2842                 return true;
2843         default:
2844                 return false;
2845         }
2846 }
2847
2848 static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length,
2849                           unsigned char *va)
2850 {
2851         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2852         struct net_device *netdev = ring_to_netdev(ring);
2853         struct sk_buff *skb;
2854
2855         ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);
2856         skb = ring->skb;
2857         if (unlikely(!skb)) {
2858                 hns3_rl_err(netdev, "alloc rx skb fail\n");
2859
2860                 u64_stats_update_begin(&ring->syncp);
2861                 ring->stats.sw_err_cnt++;
2862                 u64_stats_update_end(&ring->syncp);
2863
2864                 return -ENOMEM;
2865         }
2866
2867         trace_hns3_rx_desc(ring);
2868         prefetchw(skb->data);
2869
2870         ring->pending_buf = 1;
2871         ring->frag_num = 0;
2872         ring->tail_skb = NULL;
2873         if (length <= HNS3_RX_HEAD_SIZE) {
2874                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2875
2876                 /* We can reuse buffer as-is, just make sure it is local */
2877                 if (likely(hns3_page_is_reusable(desc_cb->priv)))
2878                         desc_cb->reuse_flag = 1;
2879                 else /* This page cannot be reused so discard it */
2880                         __page_frag_cache_drain(desc_cb->priv,
2881                                                 desc_cb->pagecnt_bias);
2882
2883                 ring_ptr_move_fw(ring, next_to_clean);
2884                 return 0;
2885         }
2886         u64_stats_update_begin(&ring->syncp);
2887         ring->stats.seg_pkt_cnt++;
2888         u64_stats_update_end(&ring->syncp);
2889
2890         ring->pull_len = eth_get_headlen(netdev, va, HNS3_RX_HEAD_SIZE);
2891         __skb_put(skb, ring->pull_len);
2892         hns3_nic_reuse_page(skb, ring->frag_num++, ring, ring->pull_len,
2893                             desc_cb);
2894         ring_ptr_move_fw(ring, next_to_clean);
2895
2896         return 0;
2897 }
2898
2899 static int hns3_add_frag(struct hns3_enet_ring *ring)
2900 {
2901         struct sk_buff *skb = ring->skb;
2902         struct sk_buff *head_skb = skb;
2903         struct sk_buff *new_skb;
2904         struct hns3_desc_cb *desc_cb;
2905         struct hns3_desc *desc;
2906         u32 bd_base_info;
2907
2908         do {
2909                 desc = &ring->desc[ring->next_to_clean];
2910                 desc_cb = &ring->desc_cb[ring->next_to_clean];
2911                 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2912                 /* make sure HW write desc complete */
2913                 dma_rmb();
2914                 if (!(bd_base_info & BIT(HNS3_RXD_VLD_B)))
2915                         return -ENXIO;
2916
2917                 if (unlikely(ring->frag_num >= MAX_SKB_FRAGS)) {
2918                         new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0);
2919                         if (unlikely(!new_skb)) {
2920                                 hns3_rl_err(ring_to_netdev(ring),
2921                                             "alloc rx fraglist skb fail\n");
2922                                 return -ENXIO;
2923                         }
2924                         ring->frag_num = 0;
2925
2926                         if (ring->tail_skb) {
2927                                 ring->tail_skb->next = new_skb;
2928                                 ring->tail_skb = new_skb;
2929                         } else {
2930                                 skb_shinfo(skb)->frag_list = new_skb;
2931                                 ring->tail_skb = new_skb;
2932                         }
2933                 }
2934
2935                 if (ring->tail_skb) {
2936                         head_skb->truesize += hns3_buf_size(ring);
2937                         head_skb->data_len += le16_to_cpu(desc->rx.size);
2938                         head_skb->len += le16_to_cpu(desc->rx.size);
2939                         skb = ring->tail_skb;
2940                 }
2941
2942                 dma_sync_single_for_cpu(ring_to_dev(ring),
2943                                 desc_cb->dma + desc_cb->page_offset,
2944                                 hns3_buf_size(ring),
2945                                 DMA_FROM_DEVICE);
2946
2947                 hns3_nic_reuse_page(skb, ring->frag_num++, ring, 0, desc_cb);
2948                 trace_hns3_rx_desc(ring);
2949                 ring_ptr_move_fw(ring, next_to_clean);
2950                 ring->pending_buf++;
2951         } while (!(bd_base_info & BIT(HNS3_RXD_FE_B)));
2952
2953         return 0;
2954 }
2955
2956 static int hns3_set_gro_and_checksum(struct hns3_enet_ring *ring,
2957                                      struct sk_buff *skb, u32 l234info,
2958                                      u32 bd_base_info, u32 ol_info)
2959 {
2960         u32 l3_type;
2961
2962         skb_shinfo(skb)->gso_size = hnae3_get_field(bd_base_info,
2963                                                     HNS3_RXD_GRO_SIZE_M,
2964                                                     HNS3_RXD_GRO_SIZE_S);
2965         /* if there is no HW GRO, do not set gro params */
2966         if (!skb_shinfo(skb)->gso_size) {
2967                 hns3_rx_checksum(ring, skb, l234info, bd_base_info, ol_info);
2968                 return 0;
2969         }
2970
2971         NAPI_GRO_CB(skb)->count = hnae3_get_field(l234info,
2972                                                   HNS3_RXD_GRO_COUNT_M,
2973                                                   HNS3_RXD_GRO_COUNT_S);
2974
2975         l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S);
2976         if (l3_type == HNS3_L3_TYPE_IPV4)
2977                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
2978         else if (l3_type == HNS3_L3_TYPE_IPV6)
2979                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
2980         else
2981                 return -EFAULT;
2982
2983         return  hns3_gro_complete(skb, l234info);
2984 }
2985
2986 static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
2987                                      struct sk_buff *skb, u32 rss_hash)
2988 {
2989         struct hnae3_handle *handle = ring->tqp->handle;
2990         enum pkt_hash_types rss_type;
2991
2992         if (rss_hash)
2993                 rss_type = handle->kinfo.rss_type;
2994         else
2995                 rss_type = PKT_HASH_TYPE_NONE;
2996
2997         skb_set_hash(skb, rss_hash, rss_type);
2998 }
2999
3000 static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb)
3001 {
3002         struct net_device *netdev = ring_to_netdev(ring);
3003         enum hns3_pkt_l2t_type l2_frame_type;
3004         u32 bd_base_info, l234info, ol_info;
3005         struct hns3_desc *desc;
3006         unsigned int len;
3007         int pre_ntc, ret;
3008
3009         /* bdinfo handled below is only valid on the last BD of the
3010          * current packet, and ring->next_to_clean indicates the first
3011          * descriptor of next packet, so need - 1 below.
3012          */
3013         pre_ntc = ring->next_to_clean ? (ring->next_to_clean - 1) :
3014                                         (ring->desc_num - 1);
3015         desc = &ring->desc[pre_ntc];
3016         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
3017         l234info = le32_to_cpu(desc->rx.l234_info);
3018         ol_info = le32_to_cpu(desc->rx.ol_info);
3019
3020         /* Based on hw strategy, the tag offloaded will be stored at
3021          * ot_vlan_tag in two layer tag case, and stored at vlan_tag
3022          * in one layer tag case.
3023          */
3024         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
3025                 u16 vlan_tag;
3026
3027                 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
3028                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
3029                                                vlan_tag);
3030         }
3031
3032         if (unlikely(!desc->rx.pkt_len || (l234info & (BIT(HNS3_RXD_TRUNCAT_B) |
3033                                   BIT(HNS3_RXD_L2E_B))))) {
3034                 u64_stats_update_begin(&ring->syncp);
3035                 if (l234info & BIT(HNS3_RXD_L2E_B))
3036                         ring->stats.l2_err++;
3037                 else
3038                         ring->stats.err_pkt_len++;
3039                 u64_stats_update_end(&ring->syncp);
3040
3041                 return -EFAULT;
3042         }
3043
3044         len = skb->len;
3045
3046         /* Do update ip stack process */
3047         skb->protocol = eth_type_trans(skb, netdev);
3048
3049         /* This is needed in order to enable forwarding support */
3050         ret = hns3_set_gro_and_checksum(ring, skb, l234info,
3051                                         bd_base_info, ol_info);
3052         if (unlikely(ret)) {
3053                 u64_stats_update_begin(&ring->syncp);
3054                 ring->stats.rx_err_cnt++;
3055                 u64_stats_update_end(&ring->syncp);
3056                 return ret;
3057         }
3058
3059         l2_frame_type = hnae3_get_field(l234info, HNS3_RXD_DMAC_M,
3060                                         HNS3_RXD_DMAC_S);
3061
3062         u64_stats_update_begin(&ring->syncp);
3063         ring->stats.rx_pkts++;
3064         ring->stats.rx_bytes += len;
3065
3066         if (l2_frame_type == HNS3_L2_TYPE_MULTICAST)
3067                 ring->stats.rx_multicast++;
3068
3069         u64_stats_update_end(&ring->syncp);
3070
3071         ring->tqp_vector->rx_group.total_bytes += len;
3072
3073         hns3_set_rx_skb_rss_type(ring, skb, le32_to_cpu(desc->rx.rss_hash));
3074         return 0;
3075 }
3076
3077 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring)
3078 {
3079         struct sk_buff *skb = ring->skb;
3080         struct hns3_desc_cb *desc_cb;
3081         struct hns3_desc *desc;
3082         unsigned int length;
3083         u32 bd_base_info;
3084         int ret;
3085
3086         desc = &ring->desc[ring->next_to_clean];
3087         desc_cb = &ring->desc_cb[ring->next_to_clean];
3088
3089         prefetch(desc);
3090
3091         length = le16_to_cpu(desc->rx.size);
3092         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
3093
3094         /* Check valid BD */
3095         if (unlikely(!(bd_base_info & BIT(HNS3_RXD_VLD_B))))
3096                 return -ENXIO;
3097
3098         if (!skb) {
3099                 ring->va = desc_cb->buf + desc_cb->page_offset;
3100
3101                 dma_sync_single_for_cpu(ring_to_dev(ring),
3102                                 desc_cb->dma + desc_cb->page_offset,
3103                                 hns3_buf_size(ring),
3104                                 DMA_FROM_DEVICE);
3105         }
3106
3107         /* Prefetch first cache line of first page
3108          * Idea is to cache few bytes of the header of the packet. Our L1 Cache
3109          * line size is 64B so need to prefetch twice to make it 128B. But in
3110          * actual we can have greater size of caches with 128B Level 1 cache
3111          * lines. In such a case, single fetch would suffice to cache in the
3112          * relevant part of the header.
3113          */
3114         net_prefetch(ring->va);
3115
3116         if (!skb) {
3117                 ret = hns3_alloc_skb(ring, length, ring->va);
3118                 skb = ring->skb;
3119
3120                 if (ret < 0) /* alloc buffer fail */
3121                         return ret;
3122                 if (!(bd_base_info & BIT(HNS3_RXD_FE_B))) { /* need add frag */
3123                         ret = hns3_add_frag(ring);
3124                         if (ret)
3125                                 return ret;
3126                 }
3127         } else {
3128                 ret = hns3_add_frag(ring);
3129                 if (ret)
3130                         return ret;
3131         }
3132
3133         /* As the head data may be changed when GRO enable, copy
3134          * the head data in after other data rx completed
3135          */
3136         if (skb->len > HNS3_RX_HEAD_SIZE)
3137                 memcpy(skb->data, ring->va,
3138                        ALIGN(ring->pull_len, sizeof(long)));
3139
3140         ret = hns3_handle_bdinfo(ring, skb);
3141         if (unlikely(ret)) {
3142                 dev_kfree_skb_any(skb);
3143                 return ret;
3144         }
3145
3146         skb_record_rx_queue(skb, ring->tqp->tqp_index);
3147         return 0;
3148 }
3149
3150 int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget,
3151                        void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
3152 {
3153 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
3154         int unused_count = hns3_desc_unused(ring);
3155         int recv_pkts = 0;
3156         int recv_bds = 0;
3157         int err, num;
3158
3159         num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
3160         num -= unused_count;
3161         unused_count -= ring->pending_buf;
3162
3163         if (num <= 0)
3164                 goto out;
3165
3166         rmb(); /* Make sure num taken effect before the other data is touched */
3167
3168         while (recv_pkts < budget && recv_bds < num) {
3169                 /* Reuse or realloc buffers */
3170                 if (unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
3171                         hns3_nic_alloc_rx_buffers(ring, unused_count);
3172                         unused_count = hns3_desc_unused(ring) -
3173                                         ring->pending_buf;
3174                 }
3175
3176                 /* Poll one pkt */
3177                 err = hns3_handle_rx_bd(ring);
3178                 /* Do not get FE for the packet or failed to alloc skb */
3179                 if (unlikely(!ring->skb || err == -ENXIO)) {
3180                         goto out;
3181                 } else if (likely(!err)) {
3182                         rx_fn(ring, ring->skb);
3183                         recv_pkts++;
3184                 }
3185
3186                 recv_bds += ring->pending_buf;
3187                 unused_count += ring->pending_buf;
3188                 ring->skb = NULL;
3189                 ring->pending_buf = 0;
3190         }
3191
3192 out:
3193         /* Make all data has been write before submit */
3194         if (unused_count > 0)
3195                 hns3_nic_alloc_rx_buffers(ring, unused_count);
3196
3197         return recv_pkts;
3198 }
3199
3200 static bool hns3_get_new_flow_lvl(struct hns3_enet_ring_group *ring_group)
3201 {
3202 #define HNS3_RX_LOW_BYTE_RATE 10000
3203 #define HNS3_RX_MID_BYTE_RATE 20000
3204 #define HNS3_RX_ULTRA_PACKET_RATE 40
3205
3206         enum hns3_flow_level_range new_flow_level;
3207         struct hns3_enet_tqp_vector *tqp_vector;
3208         int packets_per_msecs, bytes_per_msecs;
3209         u32 time_passed_ms;
3210
3211         tqp_vector = ring_group->ring->tqp_vector;
3212         time_passed_ms =
3213                 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
3214         if (!time_passed_ms)
3215                 return false;
3216
3217         do_div(ring_group->total_packets, time_passed_ms);
3218         packets_per_msecs = ring_group->total_packets;
3219
3220         do_div(ring_group->total_bytes, time_passed_ms);
3221         bytes_per_msecs = ring_group->total_bytes;
3222
3223         new_flow_level = ring_group->coal.flow_level;
3224
3225         /* Simple throttlerate management
3226          * 0-10MB/s   lower     (50000 ints/s)
3227          * 10-20MB/s   middle    (20000 ints/s)
3228          * 20-1249MB/s high      (18000 ints/s)
3229          * > 40000pps  ultra     (8000 ints/s)
3230          */
3231         switch (new_flow_level) {
3232         case HNS3_FLOW_LOW:
3233                 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
3234                         new_flow_level = HNS3_FLOW_MID;
3235                 break;
3236         case HNS3_FLOW_MID:
3237                 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
3238                         new_flow_level = HNS3_FLOW_HIGH;
3239                 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
3240                         new_flow_level = HNS3_FLOW_LOW;
3241                 break;
3242         case HNS3_FLOW_HIGH:
3243         case HNS3_FLOW_ULTRA:
3244         default:
3245                 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
3246                         new_flow_level = HNS3_FLOW_MID;
3247                 break;
3248         }
3249
3250         if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
3251             &tqp_vector->rx_group == ring_group)
3252                 new_flow_level = HNS3_FLOW_ULTRA;
3253
3254         ring_group->total_bytes = 0;
3255         ring_group->total_packets = 0;
3256         ring_group->coal.flow_level = new_flow_level;
3257
3258         return true;
3259 }
3260
3261 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
3262 {
3263         struct hns3_enet_tqp_vector *tqp_vector;
3264         u16 new_int_gl;
3265
3266         if (!ring_group->ring)
3267                 return false;
3268
3269         tqp_vector = ring_group->ring->tqp_vector;
3270         if (!tqp_vector->last_jiffies)
3271                 return false;
3272
3273         if (ring_group->total_packets == 0) {
3274                 ring_group->coal.int_gl = HNS3_INT_GL_50K;
3275                 ring_group->coal.flow_level = HNS3_FLOW_LOW;
3276                 return true;
3277         }
3278
3279         if (!hns3_get_new_flow_lvl(ring_group))
3280                 return false;
3281
3282         new_int_gl = ring_group->coal.int_gl;
3283         switch (ring_group->coal.flow_level) {
3284         case HNS3_FLOW_LOW:
3285                 new_int_gl = HNS3_INT_GL_50K;
3286                 break;
3287         case HNS3_FLOW_MID:
3288                 new_int_gl = HNS3_INT_GL_20K;
3289                 break;
3290         case HNS3_FLOW_HIGH:
3291                 new_int_gl = HNS3_INT_GL_18K;
3292                 break;
3293         case HNS3_FLOW_ULTRA:
3294                 new_int_gl = HNS3_INT_GL_8K;
3295                 break;
3296         default:
3297                 break;
3298         }
3299
3300         if (new_int_gl != ring_group->coal.int_gl) {
3301                 ring_group->coal.int_gl = new_int_gl;
3302                 return true;
3303         }
3304         return false;
3305 }
3306
3307 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
3308 {
3309         struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
3310         struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
3311         bool rx_update, tx_update;
3312
3313         /* update param every 1000ms */
3314         if (time_before(jiffies,
3315                         tqp_vector->last_jiffies + msecs_to_jiffies(1000)))
3316                 return;
3317
3318         if (rx_group->coal.gl_adapt_enable) {
3319                 rx_update = hns3_get_new_int_gl(rx_group);
3320                 if (rx_update)
3321                         hns3_set_vector_coalesce_rx_gl(tqp_vector,
3322                                                        rx_group->coal.int_gl);
3323         }
3324
3325         if (tx_group->coal.gl_adapt_enable) {
3326                 tx_update = hns3_get_new_int_gl(tx_group);
3327                 if (tx_update)
3328                         hns3_set_vector_coalesce_tx_gl(tqp_vector,
3329                                                        tx_group->coal.int_gl);
3330         }
3331
3332         tqp_vector->last_jiffies = jiffies;
3333 }
3334
3335 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
3336 {
3337         struct hns3_nic_priv *priv = netdev_priv(napi->dev);
3338         struct hns3_enet_ring *ring;
3339         int rx_pkt_total = 0;
3340
3341         struct hns3_enet_tqp_vector *tqp_vector =
3342                 container_of(napi, struct hns3_enet_tqp_vector, napi);
3343         bool clean_complete = true;
3344         int rx_budget = budget;
3345
3346         if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
3347                 napi_complete(napi);
3348                 return 0;
3349         }
3350
3351         /* Since the actual Tx work is minimal, we can give the Tx a larger
3352          * budget and be more aggressive about cleaning up the Tx descriptors.
3353          */
3354         hns3_for_each_ring(ring, tqp_vector->tx_group)
3355                 hns3_clean_tx_ring(ring);
3356
3357         /* make sure rx ring budget not smaller than 1 */
3358         if (tqp_vector->num_tqps > 1)
3359                 rx_budget = max(budget / tqp_vector->num_tqps, 1);
3360
3361         hns3_for_each_ring(ring, tqp_vector->rx_group) {
3362                 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
3363                                                     hns3_rx_skb);
3364
3365                 if (rx_cleaned >= rx_budget)
3366                         clean_complete = false;
3367
3368                 rx_pkt_total += rx_cleaned;
3369         }
3370
3371         tqp_vector->rx_group.total_packets += rx_pkt_total;
3372
3373         if (!clean_complete)
3374                 return budget;
3375
3376         if (napi_complete(napi) &&
3377             likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
3378                 hns3_update_new_int_gl(tqp_vector);
3379                 hns3_mask_vector_irq(tqp_vector, 1);
3380         }
3381
3382         return rx_pkt_total;
3383 }
3384
3385 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
3386                                       struct hnae3_ring_chain_node *head)
3387 {
3388         struct pci_dev *pdev = tqp_vector->handle->pdev;
3389         struct hnae3_ring_chain_node *cur_chain = head;
3390         struct hnae3_ring_chain_node *chain;
3391         struct hns3_enet_ring *tx_ring;
3392         struct hns3_enet_ring *rx_ring;
3393
3394         tx_ring = tqp_vector->tx_group.ring;
3395         if (tx_ring) {
3396                 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
3397                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
3398                               HNAE3_RING_TYPE_TX);
3399                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
3400                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
3401
3402                 cur_chain->next = NULL;
3403
3404                 while (tx_ring->next) {
3405                         tx_ring = tx_ring->next;
3406
3407                         chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
3408                                              GFP_KERNEL);
3409                         if (!chain)
3410                                 goto err_free_chain;
3411
3412                         cur_chain->next = chain;
3413                         chain->tqp_index = tx_ring->tqp->tqp_index;
3414                         hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
3415                                       HNAE3_RING_TYPE_TX);
3416                         hnae3_set_field(chain->int_gl_idx,
3417                                         HNAE3_RING_GL_IDX_M,
3418                                         HNAE3_RING_GL_IDX_S,
3419                                         HNAE3_RING_GL_TX);
3420
3421                         cur_chain = chain;
3422                 }
3423         }
3424
3425         rx_ring = tqp_vector->rx_group.ring;
3426         if (!tx_ring && rx_ring) {
3427                 cur_chain->next = NULL;
3428                 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
3429                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
3430                               HNAE3_RING_TYPE_RX);
3431                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
3432                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
3433
3434                 rx_ring = rx_ring->next;
3435         }
3436
3437         while (rx_ring) {
3438                 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
3439                 if (!chain)
3440                         goto err_free_chain;
3441
3442                 cur_chain->next = chain;
3443                 chain->tqp_index = rx_ring->tqp->tqp_index;
3444                 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
3445                               HNAE3_RING_TYPE_RX);
3446                 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
3447                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
3448
3449                 cur_chain = chain;
3450
3451                 rx_ring = rx_ring->next;
3452         }
3453
3454         return 0;
3455
3456 err_free_chain:
3457         cur_chain = head->next;
3458         while (cur_chain) {
3459                 chain = cur_chain->next;
3460                 devm_kfree(&pdev->dev, cur_chain);
3461                 cur_chain = chain;
3462         }
3463         head->next = NULL;
3464
3465         return -ENOMEM;
3466 }
3467
3468 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
3469                                         struct hnae3_ring_chain_node *head)
3470 {
3471         struct pci_dev *pdev = tqp_vector->handle->pdev;
3472         struct hnae3_ring_chain_node *chain_tmp, *chain;
3473
3474         chain = head->next;
3475
3476         while (chain) {
3477                 chain_tmp = chain->next;
3478                 devm_kfree(&pdev->dev, chain);
3479                 chain = chain_tmp;
3480         }
3481 }
3482
3483 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
3484                                    struct hns3_enet_ring *ring)
3485 {
3486         ring->next = group->ring;
3487         group->ring = ring;
3488
3489         group->count++;
3490 }
3491
3492 static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
3493 {
3494         struct pci_dev *pdev = priv->ae_handle->pdev;
3495         struct hns3_enet_tqp_vector *tqp_vector;
3496         int num_vectors = priv->vector_num;
3497         int numa_node;
3498         int vector_i;
3499
3500         numa_node = dev_to_node(&pdev->dev);
3501
3502         for (vector_i = 0; vector_i < num_vectors; vector_i++) {
3503                 tqp_vector = &priv->tqp_vector[vector_i];
3504                 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
3505                                 &tqp_vector->affinity_mask);
3506         }
3507 }
3508
3509 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
3510 {
3511         struct hnae3_ring_chain_node vector_ring_chain;
3512         struct hnae3_handle *h = priv->ae_handle;
3513         struct hns3_enet_tqp_vector *tqp_vector;
3514         int ret = 0;
3515         int i;
3516
3517         hns3_nic_set_cpumask(priv);
3518
3519         for (i = 0; i < priv->vector_num; i++) {
3520                 tqp_vector = &priv->tqp_vector[i];
3521                 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
3522                 tqp_vector->num_tqps = 0;
3523         }
3524
3525         for (i = 0; i < h->kinfo.num_tqps; i++) {
3526                 u16 vector_i = i % priv->vector_num;
3527                 u16 tqp_num = h->kinfo.num_tqps;
3528
3529                 tqp_vector = &priv->tqp_vector[vector_i];
3530
3531                 hns3_add_ring_to_group(&tqp_vector->tx_group,
3532                                        &priv->ring[i]);
3533
3534                 hns3_add_ring_to_group(&tqp_vector->rx_group,
3535                                        &priv->ring[i + tqp_num]);
3536
3537                 priv->ring[i].tqp_vector = tqp_vector;
3538                 priv->ring[i + tqp_num].tqp_vector = tqp_vector;
3539                 tqp_vector->num_tqps++;
3540         }
3541
3542         for (i = 0; i < priv->vector_num; i++) {
3543                 tqp_vector = &priv->tqp_vector[i];
3544
3545                 tqp_vector->rx_group.total_bytes = 0;
3546                 tqp_vector->rx_group.total_packets = 0;
3547                 tqp_vector->tx_group.total_bytes = 0;
3548                 tqp_vector->tx_group.total_packets = 0;
3549                 tqp_vector->handle = h;
3550
3551                 ret = hns3_get_vector_ring_chain(tqp_vector,
3552                                                  &vector_ring_chain);
3553                 if (ret)
3554                         goto map_ring_fail;
3555
3556                 ret = h->ae_algo->ops->map_ring_to_vector(h,
3557                         tqp_vector->vector_irq, &vector_ring_chain);
3558
3559                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3560
3561                 if (ret)
3562                         goto map_ring_fail;
3563
3564                 netif_napi_add(priv->netdev, &tqp_vector->napi,
3565                                hns3_nic_common_poll, NAPI_POLL_WEIGHT);
3566         }
3567
3568         return 0;
3569
3570 map_ring_fail:
3571         while (i--)
3572                 netif_napi_del(&priv->tqp_vector[i].napi);
3573
3574         return ret;
3575 }
3576
3577 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
3578 {
3579 #define HNS3_VECTOR_PF_MAX_NUM          64
3580
3581         struct hnae3_handle *h = priv->ae_handle;
3582         struct hns3_enet_tqp_vector *tqp_vector;
3583         struct hnae3_vector_info *vector;
3584         struct pci_dev *pdev = h->pdev;
3585         u16 tqp_num = h->kinfo.num_tqps;
3586         u16 vector_num;
3587         int ret = 0;
3588         u16 i;
3589
3590         /* RSS size, cpu online and vector_num should be the same */
3591         /* Should consider 2p/4p later */
3592         vector_num = min_t(u16, num_online_cpus(), tqp_num);
3593         vector_num = min_t(u16, vector_num, HNS3_VECTOR_PF_MAX_NUM);
3594
3595         vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
3596                               GFP_KERNEL);
3597         if (!vector)
3598                 return -ENOMEM;
3599
3600         /* save the actual available vector number */
3601         vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
3602
3603         priv->vector_num = vector_num;
3604         priv->tqp_vector = (struct hns3_enet_tqp_vector *)
3605                 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
3606                              GFP_KERNEL);
3607         if (!priv->tqp_vector) {
3608                 ret = -ENOMEM;
3609                 goto out;
3610         }
3611
3612         for (i = 0; i < priv->vector_num; i++) {
3613                 tqp_vector = &priv->tqp_vector[i];
3614                 tqp_vector->idx = i;
3615                 tqp_vector->mask_addr = vector[i].io_addr;
3616                 tqp_vector->vector_irq = vector[i].vector;
3617                 hns3_vector_gl_rl_init(tqp_vector, priv);
3618         }
3619
3620 out:
3621         devm_kfree(&pdev->dev, vector);
3622         return ret;
3623 }
3624
3625 static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
3626 {
3627         group->ring = NULL;
3628         group->count = 0;
3629 }
3630
3631 static void hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
3632 {
3633         struct hnae3_ring_chain_node vector_ring_chain;
3634         struct hnae3_handle *h = priv->ae_handle;
3635         struct hns3_enet_tqp_vector *tqp_vector;
3636         int i;
3637
3638         for (i = 0; i < priv->vector_num; i++) {
3639                 tqp_vector = &priv->tqp_vector[i];
3640
3641                 if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring)
3642                         continue;
3643
3644                 /* Since the mapping can be overwritten, when fail to get the
3645                  * chain between vector and ring, we should go on to deal with
3646                  * the remaining options.
3647                  */
3648                 if (hns3_get_vector_ring_chain(tqp_vector, &vector_ring_chain))
3649                         dev_warn(priv->dev, "failed to get ring chain\n");
3650
3651                 h->ae_algo->ops->unmap_ring_from_vector(h,
3652                         tqp_vector->vector_irq, &vector_ring_chain);
3653
3654                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3655
3656                 hns3_clear_ring_group(&tqp_vector->rx_group);
3657                 hns3_clear_ring_group(&tqp_vector->tx_group);
3658                 netif_napi_del(&priv->tqp_vector[i].napi);
3659         }
3660 }
3661
3662 static void hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
3663 {
3664         struct hnae3_handle *h = priv->ae_handle;
3665         struct pci_dev *pdev = h->pdev;
3666         int i, ret;
3667
3668         for (i = 0; i < priv->vector_num; i++) {
3669                 struct hns3_enet_tqp_vector *tqp_vector;
3670
3671                 tqp_vector = &priv->tqp_vector[i];
3672                 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
3673                 if (ret)
3674                         return;
3675         }
3676
3677         devm_kfree(&pdev->dev, priv->tqp_vector);
3678 }
3679
3680 static void hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
3681                               unsigned int ring_type)
3682 {
3683         int queue_num = priv->ae_handle->kinfo.num_tqps;
3684         struct hns3_enet_ring *ring;
3685         int desc_num;
3686
3687         if (ring_type == HNAE3_RING_TYPE_TX) {
3688                 ring = &priv->ring[q->tqp_index];
3689                 desc_num = priv->ae_handle->kinfo.num_tx_desc;
3690                 ring->queue_index = q->tqp_index;
3691         } else {
3692                 ring = &priv->ring[q->tqp_index + queue_num];
3693                 desc_num = priv->ae_handle->kinfo.num_rx_desc;
3694                 ring->queue_index = q->tqp_index;
3695         }
3696
3697         hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
3698
3699         ring->tqp = q;
3700         ring->desc = NULL;
3701         ring->desc_cb = NULL;
3702         ring->dev = priv->dev;
3703         ring->desc_dma_addr = 0;
3704         ring->buf_size = q->buf_size;
3705         ring->desc_num = desc_num;
3706         ring->next_to_use = 0;
3707         ring->next_to_clean = 0;
3708         ring->last_to_use = 0;
3709 }
3710
3711 static void hns3_queue_to_ring(struct hnae3_queue *tqp,
3712                                struct hns3_nic_priv *priv)
3713 {
3714         hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
3715         hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
3716 }
3717
3718 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
3719 {
3720         struct hnae3_handle *h = priv->ae_handle;
3721         struct pci_dev *pdev = h->pdev;
3722         int i;
3723
3724         priv->ring = devm_kzalloc(&pdev->dev,
3725                                   array3_size(h->kinfo.num_tqps,
3726                                               sizeof(*priv->ring), 2),
3727                                   GFP_KERNEL);
3728         if (!priv->ring)
3729                 return -ENOMEM;
3730
3731         for (i = 0; i < h->kinfo.num_tqps; i++)
3732                 hns3_queue_to_ring(h->kinfo.tqp[i], priv);
3733
3734         return 0;
3735 }
3736
3737 static void hns3_put_ring_config(struct hns3_nic_priv *priv)
3738 {
3739         if (!priv->ring)
3740                 return;
3741
3742         devm_kfree(priv->dev, priv->ring);
3743         priv->ring = NULL;
3744 }
3745
3746 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
3747 {
3748         int ret;
3749
3750         if (ring->desc_num <= 0 || ring->buf_size <= 0)
3751                 return -EINVAL;
3752
3753         ring->desc_cb = devm_kcalloc(ring_to_dev(ring), ring->desc_num,
3754                                      sizeof(ring->desc_cb[0]), GFP_KERNEL);
3755         if (!ring->desc_cb) {
3756                 ret = -ENOMEM;
3757                 goto out;
3758         }
3759
3760         ret = hns3_alloc_desc(ring);
3761         if (ret)
3762                 goto out_with_desc_cb;
3763
3764         if (!HNAE3_IS_TX_RING(ring)) {
3765                 ret = hns3_alloc_ring_buffers(ring);
3766                 if (ret)
3767                         goto out_with_desc;
3768         }
3769
3770         return 0;
3771
3772 out_with_desc:
3773         hns3_free_desc(ring);
3774 out_with_desc_cb:
3775         devm_kfree(ring_to_dev(ring), ring->desc_cb);
3776         ring->desc_cb = NULL;
3777 out:
3778         return ret;
3779 }
3780
3781 void hns3_fini_ring(struct hns3_enet_ring *ring)
3782 {
3783         hns3_free_desc(ring);
3784         devm_kfree(ring_to_dev(ring), ring->desc_cb);
3785         ring->desc_cb = NULL;
3786         ring->next_to_clean = 0;
3787         ring->next_to_use = 0;
3788         ring->last_to_use = 0;
3789         ring->pending_buf = 0;
3790         if (ring->skb) {
3791                 dev_kfree_skb_any(ring->skb);
3792                 ring->skb = NULL;
3793         }
3794 }
3795
3796 static int hns3_buf_size2type(u32 buf_size)
3797 {
3798         int bd_size_type;
3799
3800         switch (buf_size) {
3801         case 512:
3802                 bd_size_type = HNS3_BD_SIZE_512_TYPE;
3803                 break;
3804         case 1024:
3805                 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
3806                 break;
3807         case 2048:
3808                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3809                 break;
3810         case 4096:
3811                 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
3812                 break;
3813         default:
3814                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3815         }
3816
3817         return bd_size_type;
3818 }
3819
3820 static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
3821 {
3822         dma_addr_t dma = ring->desc_dma_addr;
3823         struct hnae3_queue *q = ring->tqp;
3824
3825         if (!HNAE3_IS_TX_RING(ring)) {
3826                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG, (u32)dma);
3827                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
3828                                (u32)((dma >> 31) >> 1));
3829
3830                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
3831                                hns3_buf_size2type(ring->buf_size));
3832                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
3833                                ring->desc_num / 8 - 1);
3834
3835         } else {
3836                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
3837                                (u32)dma);
3838                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
3839                                (u32)((dma >> 31) >> 1));
3840
3841                 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
3842                                ring->desc_num / 8 - 1);
3843         }
3844 }
3845
3846 static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
3847 {
3848         struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
3849         int i;
3850
3851         for (i = 0; i < HNAE3_MAX_TC; i++) {
3852                 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3853                 int j;
3854
3855                 if (!tc_info->enable)
3856                         continue;
3857
3858                 for (j = 0; j < tc_info->tqp_count; j++) {
3859                         struct hnae3_queue *q;
3860
3861                         q = priv->ring[tc_info->tqp_offset + j].tqp;
3862                         hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG,
3863                                        tc_info->tc);
3864                 }
3865         }
3866 }
3867
3868 int hns3_init_all_ring(struct hns3_nic_priv *priv)
3869 {
3870         struct hnae3_handle *h = priv->ae_handle;
3871         int ring_num = h->kinfo.num_tqps * 2;
3872         int i, j;
3873         int ret;
3874
3875         for (i = 0; i < ring_num; i++) {
3876                 ret = hns3_alloc_ring_memory(&priv->ring[i]);
3877                 if (ret) {
3878                         dev_err(priv->dev,
3879                                 "Alloc ring memory fail! ret=%d\n", ret);
3880                         goto out_when_alloc_ring_memory;
3881                 }
3882
3883                 u64_stats_init(&priv->ring[i].syncp);
3884         }
3885
3886         return 0;
3887
3888 out_when_alloc_ring_memory:
3889         for (j = i - 1; j >= 0; j--)
3890                 hns3_fini_ring(&priv->ring[j]);
3891
3892         return -ENOMEM;
3893 }
3894
3895 int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
3896 {
3897         struct hnae3_handle *h = priv->ae_handle;
3898         int i;
3899
3900         for (i = 0; i < h->kinfo.num_tqps; i++) {
3901                 hns3_fini_ring(&priv->ring[i]);
3902                 hns3_fini_ring(&priv->ring[i + h->kinfo.num_tqps]);
3903         }
3904         return 0;
3905 }
3906
3907 /* Set mac addr if it is configured. or leave it to the AE driver */
3908 static int hns3_init_mac_addr(struct net_device *netdev)
3909 {
3910         struct hns3_nic_priv *priv = netdev_priv(netdev);
3911         struct hnae3_handle *h = priv->ae_handle;
3912         u8 mac_addr_temp[ETH_ALEN];
3913         int ret = 0;
3914
3915         if (h->ae_algo->ops->get_mac_addr)
3916                 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3917
3918         /* Check if the MAC address is valid, if not get a random one */
3919         if (!is_valid_ether_addr(mac_addr_temp)) {
3920                 eth_hw_addr_random(netdev);
3921                 dev_warn(priv->dev, "using random MAC address %pM\n",
3922                          netdev->dev_addr);
3923         } else if (!ether_addr_equal(netdev->dev_addr, mac_addr_temp)) {
3924                 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3925                 ether_addr_copy(netdev->perm_addr, mac_addr_temp);
3926         } else {
3927                 return 0;
3928         }
3929
3930         if (h->ae_algo->ops->set_mac_addr)
3931                 ret = h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
3932
3933         return ret;
3934 }
3935
3936 static int hns3_init_phy(struct net_device *netdev)
3937 {
3938         struct hnae3_handle *h = hns3_get_handle(netdev);
3939         int ret = 0;
3940
3941         if (h->ae_algo->ops->mac_connect_phy)
3942                 ret = h->ae_algo->ops->mac_connect_phy(h);
3943
3944         return ret;
3945 }
3946
3947 static void hns3_uninit_phy(struct net_device *netdev)
3948 {
3949         struct hnae3_handle *h = hns3_get_handle(netdev);
3950
3951         if (h->ae_algo->ops->mac_disconnect_phy)
3952                 h->ae_algo->ops->mac_disconnect_phy(h);
3953 }
3954
3955 static void hns3_del_all_fd_rules(struct net_device *netdev, bool clear_list)
3956 {
3957         struct hnae3_handle *h = hns3_get_handle(netdev);
3958
3959         if (h->ae_algo->ops->del_all_fd_entries)
3960                 h->ae_algo->ops->del_all_fd_entries(h, clear_list);
3961 }
3962
3963 static int hns3_client_start(struct hnae3_handle *handle)
3964 {
3965         if (!handle->ae_algo->ops->client_start)
3966                 return 0;
3967
3968         return handle->ae_algo->ops->client_start(handle);
3969 }
3970
3971 static void hns3_client_stop(struct hnae3_handle *handle)
3972 {
3973         if (!handle->ae_algo->ops->client_stop)
3974                 return;
3975
3976         handle->ae_algo->ops->client_stop(handle);
3977 }
3978
3979 static void hns3_info_show(struct hns3_nic_priv *priv)
3980 {
3981         struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
3982
3983         dev_info(priv->dev, "MAC address: %pM\n", priv->netdev->dev_addr);
3984         dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps);
3985         dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size);
3986         dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size);
3987         dev_info(priv->dev, "RX buffer length: %u\n", kinfo->rx_buf_len);
3988         dev_info(priv->dev, "Desc num per TX queue: %u\n", kinfo->num_tx_desc);
3989         dev_info(priv->dev, "Desc num per RX queue: %u\n", kinfo->num_rx_desc);
3990         dev_info(priv->dev, "Total number of enabled TCs: %u\n", kinfo->num_tc);
3991         dev_info(priv->dev, "Max mtu size: %u\n", priv->netdev->max_mtu);
3992 }
3993
3994 static int hns3_client_init(struct hnae3_handle *handle)
3995 {
3996         struct pci_dev *pdev = handle->pdev;
3997         u16 alloc_tqps, max_rss_size;
3998         struct hns3_nic_priv *priv;
3999         struct net_device *netdev;
4000         int ret;
4001
4002         handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,
4003                                                     &max_rss_size);
4004         netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);
4005         if (!netdev)
4006                 return -ENOMEM;
4007
4008         priv = netdev_priv(netdev);
4009         priv->dev = &pdev->dev;
4010         priv->netdev = netdev;
4011         priv->ae_handle = handle;
4012         priv->tx_timeout_count = 0;
4013         set_bit(HNS3_NIC_STATE_DOWN, &priv->state);
4014
4015         handle->msg_enable = netif_msg_init(debug, DEFAULT_MSG_LEVEL);
4016
4017         handle->kinfo.netdev = netdev;
4018         handle->priv = (void *)priv;
4019
4020         hns3_init_mac_addr(netdev);
4021
4022         hns3_set_default_feature(netdev);
4023
4024         netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
4025         netdev->priv_flags |= IFF_UNICAST_FLT;
4026         netdev->netdev_ops = &hns3_nic_netdev_ops;
4027         SET_NETDEV_DEV(netdev, &pdev->dev);
4028         hns3_ethtool_set_ops(netdev);
4029
4030         /* Carrier off reporting is important to ethtool even BEFORE open */
4031         netif_carrier_off(netdev);
4032
4033         ret = hns3_get_ring_config(priv);
4034         if (ret) {
4035                 ret = -ENOMEM;
4036                 goto out_get_ring_cfg;
4037         }
4038
4039         ret = hns3_nic_alloc_vector_data(priv);
4040         if (ret) {
4041                 ret = -ENOMEM;
4042                 goto out_alloc_vector_data;
4043         }
4044
4045         ret = hns3_nic_init_vector_data(priv);
4046         if (ret) {
4047                 ret = -ENOMEM;
4048                 goto out_init_vector_data;
4049         }
4050
4051         ret = hns3_init_all_ring(priv);
4052         if (ret) {
4053                 ret = -ENOMEM;
4054                 goto out_init_ring;
4055         }
4056
4057         ret = hns3_init_phy(netdev);
4058         if (ret)
4059                 goto out_init_phy;
4060
4061         ret = register_netdev(netdev);
4062         if (ret) {
4063                 dev_err(priv->dev, "probe register netdev fail!\n");
4064                 goto out_reg_netdev_fail;
4065         }
4066
4067         /* the device can work without cpu rmap, only aRFS needs it */
4068         ret = hns3_set_rx_cpu_rmap(netdev);
4069         if (ret)
4070                 dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret);
4071
4072         ret = hns3_nic_init_irq(priv);
4073         if (ret) {
4074                 dev_err(priv->dev, "init irq failed! ret=%d\n", ret);
4075                 hns3_free_rx_cpu_rmap(netdev);
4076                 goto out_init_irq_fail;
4077         }
4078
4079         ret = hns3_client_start(handle);
4080         if (ret) {
4081                 dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);
4082                 goto out_client_start;
4083         }
4084
4085         hns3_dcbnl_setup(handle);
4086
4087         hns3_dbg_init(handle);
4088
4089         /* MTU range: (ETH_MIN_MTU(kernel default) - 9702) */
4090         netdev->max_mtu = HNS3_MAX_MTU;
4091
4092         set_bit(HNS3_NIC_STATE_INITED, &priv->state);
4093
4094         if (netif_msg_drv(handle))
4095                 hns3_info_show(priv);
4096
4097         return ret;
4098
4099 out_client_start:
4100         hns3_free_rx_cpu_rmap(netdev);
4101         hns3_nic_uninit_irq(priv);
4102 out_init_irq_fail:
4103         unregister_netdev(netdev);
4104 out_reg_netdev_fail:
4105         hns3_uninit_phy(netdev);
4106 out_init_phy:
4107         hns3_uninit_all_ring(priv);
4108 out_init_ring:
4109         hns3_nic_uninit_vector_data(priv);
4110 out_init_vector_data:
4111         hns3_nic_dealloc_vector_data(priv);
4112 out_alloc_vector_data:
4113         priv->ring = NULL;
4114 out_get_ring_cfg:
4115         priv->ae_handle = NULL;
4116         free_netdev(netdev);
4117         return ret;
4118 }
4119
4120 static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
4121 {
4122         struct net_device *netdev = handle->kinfo.netdev;
4123         struct hns3_nic_priv *priv = netdev_priv(netdev);
4124         int ret;
4125
4126         if (netdev->reg_state != NETREG_UNINITIALIZED)
4127                 unregister_netdev(netdev);
4128
4129         hns3_client_stop(handle);
4130
4131         hns3_uninit_phy(netdev);
4132
4133         if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
4134                 netdev_warn(netdev, "already uninitialized\n");
4135                 goto out_netdev_free;
4136         }
4137
4138         hns3_free_rx_cpu_rmap(netdev);
4139
4140         hns3_nic_uninit_irq(priv);
4141
4142         hns3_del_all_fd_rules(netdev, true);
4143
4144         hns3_clear_all_ring(handle, true);
4145
4146         hns3_nic_uninit_vector_data(priv);
4147
4148         hns3_nic_dealloc_vector_data(priv);
4149
4150         ret = hns3_uninit_all_ring(priv);
4151         if (ret)
4152                 netdev_err(netdev, "uninit ring error\n");
4153
4154         hns3_put_ring_config(priv);
4155
4156 out_netdev_free:
4157         hns3_dbg_uninit(handle);
4158         free_netdev(netdev);
4159 }
4160
4161 static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
4162 {
4163         struct net_device *netdev = handle->kinfo.netdev;
4164
4165         if (!netdev)
4166                 return;
4167
4168         if (linkup) {
4169                 netif_tx_wake_all_queues(netdev);
4170                 netif_carrier_on(netdev);
4171                 if (netif_msg_link(handle))
4172                         netdev_info(netdev, "link up\n");
4173         } else {
4174                 netif_carrier_off(netdev);
4175                 netif_tx_stop_all_queues(netdev);
4176                 if (netif_msg_link(handle))
4177                         netdev_info(netdev, "link down\n");
4178         }
4179 }
4180
4181 static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
4182 {
4183         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
4184         struct net_device *ndev = kinfo->netdev;
4185
4186         if (tc > HNAE3_MAX_TC)
4187                 return -EINVAL;
4188
4189         if (!ndev)
4190                 return -ENODEV;
4191
4192         return hns3_nic_set_real_num_queue(ndev);
4193 }
4194
4195 static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
4196 {
4197         while (ring->next_to_clean != ring->next_to_use) {
4198                 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
4199                 hns3_free_buffer_detach(ring, ring->next_to_clean);
4200                 ring_ptr_move_fw(ring, next_to_clean);
4201         }
4202
4203         ring->pending_buf = 0;
4204 }
4205
4206 static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
4207 {
4208         struct hns3_desc_cb res_cbs;
4209         int ret;
4210
4211         while (ring->next_to_use != ring->next_to_clean) {
4212                 /* When a buffer is not reused, it's memory has been
4213                  * freed in hns3_handle_rx_bd or will be freed by
4214                  * stack, so we need to replace the buffer here.
4215                  */
4216                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
4217                         ret = hns3_alloc_and_map_buffer(ring, &res_cbs);
4218                         if (ret) {
4219                                 u64_stats_update_begin(&ring->syncp);
4220                                 ring->stats.sw_err_cnt++;
4221                                 u64_stats_update_end(&ring->syncp);
4222                                 /* if alloc new buffer fail, exit directly
4223                                  * and reclear in up flow.
4224                                  */
4225                                 netdev_warn(ring_to_netdev(ring),
4226                                             "reserve buffer map failed, ret = %d\n",
4227                                             ret);
4228                                 return ret;
4229                         }
4230                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
4231                 }
4232                 ring_ptr_move_fw(ring, next_to_use);
4233         }
4234
4235         /* Free the pending skb in rx ring */
4236         if (ring->skb) {
4237                 dev_kfree_skb_any(ring->skb);
4238                 ring->skb = NULL;
4239                 ring->pending_buf = 0;
4240         }
4241
4242         return 0;
4243 }
4244
4245 static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
4246 {
4247         while (ring->next_to_use != ring->next_to_clean) {
4248                 /* When a buffer is not reused, it's memory has been
4249                  * freed in hns3_handle_rx_bd or will be freed by
4250                  * stack, so only need to unmap the buffer here.
4251                  */
4252                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
4253                         hns3_unmap_buffer(ring,
4254                                           &ring->desc_cb[ring->next_to_use]);
4255                         ring->desc_cb[ring->next_to_use].dma = 0;
4256                 }
4257
4258                 ring_ptr_move_fw(ring, next_to_use);
4259         }
4260 }
4261
4262 static void hns3_clear_all_ring(struct hnae3_handle *h, bool force)
4263 {
4264         struct net_device *ndev = h->kinfo.netdev;
4265         struct hns3_nic_priv *priv = netdev_priv(ndev);
4266         u32 i;
4267
4268         for (i = 0; i < h->kinfo.num_tqps; i++) {
4269                 struct hns3_enet_ring *ring;
4270
4271                 ring = &priv->ring[i];
4272                 hns3_clear_tx_ring(ring);
4273
4274                 ring = &priv->ring[i + h->kinfo.num_tqps];
4275                 /* Continue to clear other rings even if clearing some
4276                  * rings failed.
4277                  */
4278                 if (force)
4279                         hns3_force_clear_rx_ring(ring);
4280                 else
4281                         hns3_clear_rx_ring(ring);
4282         }
4283 }
4284
4285 int hns3_nic_reset_all_ring(struct hnae3_handle *h)
4286 {
4287         struct net_device *ndev = h->kinfo.netdev;
4288         struct hns3_nic_priv *priv = netdev_priv(ndev);
4289         struct hns3_enet_ring *rx_ring;
4290         int i, j;
4291         int ret;
4292
4293         for (i = 0; i < h->kinfo.num_tqps; i++) {
4294                 ret = h->ae_algo->ops->reset_queue(h, i);
4295                 if (ret)
4296                         return ret;
4297
4298                 hns3_init_ring_hw(&priv->ring[i]);
4299
4300                 /* We need to clear tx ring here because self test will
4301                  * use the ring and will not run down before up
4302                  */
4303                 hns3_clear_tx_ring(&priv->ring[i]);
4304                 priv->ring[i].next_to_clean = 0;
4305                 priv->ring[i].next_to_use = 0;
4306                 priv->ring[i].last_to_use = 0;
4307
4308                 rx_ring = &priv->ring[i + h->kinfo.num_tqps];
4309                 hns3_init_ring_hw(rx_ring);
4310                 ret = hns3_clear_rx_ring(rx_ring);
4311                 if (ret)
4312                         return ret;
4313
4314                 /* We can not know the hardware head and tail when this
4315                  * function is called in reset flow, so we reuse all desc.
4316                  */
4317                 for (j = 0; j < rx_ring->desc_num; j++)
4318                         hns3_reuse_buffer(rx_ring, j);
4319
4320                 rx_ring->next_to_clean = 0;
4321                 rx_ring->next_to_use = 0;
4322         }
4323
4324         hns3_init_tx_ring_tc(priv);
4325
4326         return 0;
4327 }
4328
4329 static void hns3_store_coal(struct hns3_nic_priv *priv)
4330 {
4331         /* ethtool only support setting and querying one coal
4332          * configuration for now, so save the vector 0' coal
4333          * configuration here in order to restore it.
4334          */
4335         memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal,
4336                sizeof(struct hns3_enet_coalesce));
4337         memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal,
4338                sizeof(struct hns3_enet_coalesce));
4339 }
4340
4341 static void hns3_restore_coal(struct hns3_nic_priv *priv)
4342 {
4343         u16 vector_num = priv->vector_num;
4344         int i;
4345
4346         for (i = 0; i < vector_num; i++) {
4347                 memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal,
4348                        sizeof(struct hns3_enet_coalesce));
4349                 memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal,
4350                        sizeof(struct hns3_enet_coalesce));
4351         }
4352 }
4353
4354 static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
4355 {
4356         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
4357         struct net_device *ndev = kinfo->netdev;
4358         struct hns3_nic_priv *priv = netdev_priv(ndev);
4359
4360         if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state))
4361                 return 0;
4362
4363         if (!netif_running(ndev))
4364                 return 0;
4365
4366         return hns3_nic_net_stop(ndev);
4367 }
4368
4369 static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
4370 {
4371         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
4372         struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
4373         int ret = 0;
4374
4375         clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
4376
4377         if (netif_running(kinfo->netdev)) {
4378                 ret = hns3_nic_net_open(kinfo->netdev);
4379                 if (ret) {
4380                         set_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
4381                         netdev_err(kinfo->netdev,
4382                                    "net up fail, ret=%d!\n", ret);
4383                         return ret;
4384                 }
4385         }
4386
4387         return ret;
4388 }
4389
4390 static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
4391 {
4392         struct net_device *netdev = handle->kinfo.netdev;
4393         struct hns3_nic_priv *priv = netdev_priv(netdev);
4394         int ret;
4395
4396         /* Carrier off reporting is important to ethtool even BEFORE open */
4397         netif_carrier_off(netdev);
4398
4399         ret = hns3_get_ring_config(priv);
4400         if (ret)
4401                 return ret;
4402
4403         ret = hns3_nic_alloc_vector_data(priv);
4404         if (ret)
4405                 goto err_put_ring;
4406
4407         hns3_restore_coal(priv);
4408
4409         ret = hns3_nic_init_vector_data(priv);
4410         if (ret)
4411                 goto err_dealloc_vector;
4412
4413         ret = hns3_init_all_ring(priv);
4414         if (ret)
4415                 goto err_uninit_vector;
4416
4417         /* the device can work without cpu rmap, only aRFS needs it */
4418         ret = hns3_set_rx_cpu_rmap(netdev);
4419         if (ret)
4420                 dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret);
4421
4422         ret = hns3_nic_init_irq(priv);
4423         if (ret) {
4424                 dev_err(priv->dev, "init irq failed! ret=%d\n", ret);
4425                 hns3_free_rx_cpu_rmap(netdev);
4426                 goto err_init_irq_fail;
4427         }
4428
4429         if (!hns3_is_phys_func(handle->pdev))
4430                 hns3_init_mac_addr(netdev);
4431
4432         ret = hns3_client_start(handle);
4433         if (ret) {
4434                 dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);
4435                 goto err_client_start_fail;
4436         }
4437
4438         set_bit(HNS3_NIC_STATE_INITED, &priv->state);
4439
4440         return ret;
4441
4442 err_client_start_fail:
4443         hns3_free_rx_cpu_rmap(netdev);
4444         hns3_nic_uninit_irq(priv);
4445 err_init_irq_fail:
4446         hns3_uninit_all_ring(priv);
4447 err_uninit_vector:
4448         hns3_nic_uninit_vector_data(priv);
4449 err_dealloc_vector:
4450         hns3_nic_dealloc_vector_data(priv);
4451 err_put_ring:
4452         hns3_put_ring_config(priv);
4453
4454         return ret;
4455 }
4456
4457 static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
4458 {
4459         struct net_device *netdev = handle->kinfo.netdev;
4460         struct hns3_nic_priv *priv = netdev_priv(netdev);
4461         int ret;
4462
4463         if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
4464                 netdev_warn(netdev, "already uninitialized\n");
4465                 return 0;
4466         }
4467
4468         hns3_free_rx_cpu_rmap(netdev);
4469         hns3_nic_uninit_irq(priv);
4470         hns3_clear_all_ring(handle, true);
4471         hns3_reset_tx_queue(priv->ae_handle);
4472
4473         hns3_nic_uninit_vector_data(priv);
4474
4475         hns3_store_coal(priv);
4476
4477         hns3_nic_dealloc_vector_data(priv);
4478
4479         ret = hns3_uninit_all_ring(priv);
4480         if (ret)
4481                 netdev_err(netdev, "uninit ring error\n");
4482
4483         hns3_put_ring_config(priv);
4484
4485         return ret;
4486 }
4487
4488 static int hns3_reset_notify(struct hnae3_handle *handle,
4489                              enum hnae3_reset_notify_type type)
4490 {
4491         int ret = 0;
4492
4493         switch (type) {
4494         case HNAE3_UP_CLIENT:
4495                 ret = hns3_reset_notify_up_enet(handle);
4496                 break;
4497         case HNAE3_DOWN_CLIENT:
4498                 ret = hns3_reset_notify_down_enet(handle);
4499                 break;
4500         case HNAE3_INIT_CLIENT:
4501                 ret = hns3_reset_notify_init_enet(handle);
4502                 break;
4503         case HNAE3_UNINIT_CLIENT:
4504                 ret = hns3_reset_notify_uninit_enet(handle);
4505                 break;
4506         default:
4507                 break;
4508         }
4509
4510         return ret;
4511 }
4512
4513 static int hns3_change_channels(struct hnae3_handle *handle, u32 new_tqp_num,
4514                                 bool rxfh_configured)
4515 {
4516         int ret;
4517
4518         ret = handle->ae_algo->ops->set_channels(handle, new_tqp_num,
4519                                                  rxfh_configured);
4520         if (ret) {
4521                 dev_err(&handle->pdev->dev,
4522                         "Change tqp num(%u) fail.\n", new_tqp_num);
4523                 return ret;
4524         }
4525
4526         ret = hns3_reset_notify(handle, HNAE3_INIT_CLIENT);
4527         if (ret)
4528                 return ret;
4529
4530         ret =  hns3_reset_notify(handle, HNAE3_UP_CLIENT);
4531         if (ret)
4532                 hns3_reset_notify(handle, HNAE3_UNINIT_CLIENT);
4533
4534         return ret;
4535 }
4536
4537 int hns3_set_channels(struct net_device *netdev,
4538                       struct ethtool_channels *ch)
4539 {
4540         struct hnae3_handle *h = hns3_get_handle(netdev);
4541         struct hnae3_knic_private_info *kinfo = &h->kinfo;
4542         bool rxfh_configured = netif_is_rxfh_configured(netdev);
4543         u32 new_tqp_num = ch->combined_count;
4544         u16 org_tqp_num;
4545         int ret;
4546
4547         if (hns3_nic_resetting(netdev))
4548                 return -EBUSY;
4549
4550         if (ch->rx_count || ch->tx_count)
4551                 return -EINVAL;
4552
4553         if (new_tqp_num > hns3_get_max_available_channels(h) ||
4554             new_tqp_num < 1) {
4555                 dev_err(&netdev->dev,
4556                         "Change tqps fail, the tqp range is from 1 to %u",
4557                         hns3_get_max_available_channels(h));
4558                 return -EINVAL;
4559         }
4560
4561         if (kinfo->rss_size == new_tqp_num)
4562                 return 0;
4563
4564         netif_dbg(h, drv, netdev,
4565                   "set channels: tqp_num=%u, rxfh=%d\n",
4566                   new_tqp_num, rxfh_configured);
4567
4568         ret = hns3_reset_notify(h, HNAE3_DOWN_CLIENT);
4569         if (ret)
4570                 return ret;
4571
4572         ret = hns3_reset_notify(h, HNAE3_UNINIT_CLIENT);
4573         if (ret)
4574                 return ret;
4575
4576         org_tqp_num = h->kinfo.num_tqps;
4577         ret = hns3_change_channels(h, new_tqp_num, rxfh_configured);
4578         if (ret) {
4579                 int ret1;
4580
4581                 netdev_warn(netdev,
4582                             "Change channels fail, revert to old value\n");
4583                 ret1 = hns3_change_channels(h, org_tqp_num, rxfh_configured);
4584                 if (ret1) {
4585                         netdev_err(netdev,
4586                                    "revert to old channel fail\n");
4587                         return ret1;
4588                 }
4589
4590                 return ret;
4591         }
4592
4593         return 0;
4594 }
4595
4596 static const struct hns3_hw_error_info hns3_hw_err[] = {
4597         { .type = HNAE3_PPU_POISON_ERROR,
4598           .msg = "PPU poison" },
4599         { .type = HNAE3_CMDQ_ECC_ERROR,
4600           .msg = "IMP CMDQ error" },
4601         { .type = HNAE3_IMP_RD_POISON_ERROR,
4602           .msg = "IMP RD poison" },
4603 };
4604
4605 static void hns3_process_hw_error(struct hnae3_handle *handle,
4606                                   enum hnae3_hw_error_type type)
4607 {
4608         int i;
4609
4610         for (i = 0; i < ARRAY_SIZE(hns3_hw_err); i++) {
4611                 if (hns3_hw_err[i].type == type) {
4612                         dev_err(&handle->pdev->dev, "Detected %s!\n",
4613                                 hns3_hw_err[i].msg);
4614                         break;
4615                 }
4616         }
4617 }
4618
4619 static const struct hnae3_client_ops client_ops = {
4620         .init_instance = hns3_client_init,
4621         .uninit_instance = hns3_client_uninit,
4622         .link_status_change = hns3_link_status_change,
4623         .setup_tc = hns3_client_setup_tc,
4624         .reset_notify = hns3_reset_notify,
4625         .process_hw_error = hns3_process_hw_error,
4626 };
4627
4628 /* hns3_init_module - Driver registration routine
4629  * hns3_init_module is the first routine called when the driver is
4630  * loaded. All it does is register with the PCI subsystem.
4631  */
4632 static int __init hns3_init_module(void)
4633 {
4634         int ret;
4635
4636         pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
4637         pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
4638
4639         client.type = HNAE3_CLIENT_KNIC;
4640         snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH, "%s",
4641                  hns3_driver_name);
4642
4643         client.ops = &client_ops;
4644
4645         INIT_LIST_HEAD(&client.node);
4646
4647         hns3_dbg_register_debugfs(hns3_driver_name);
4648
4649         ret = hnae3_register_client(&client);
4650         if (ret)
4651                 goto err_reg_client;
4652
4653         ret = pci_register_driver(&hns3_driver);
4654         if (ret)
4655                 goto err_reg_driver;
4656
4657         return ret;
4658
4659 err_reg_driver:
4660         hnae3_unregister_client(&client);
4661 err_reg_client:
4662         hns3_dbg_unregister_debugfs();
4663         return ret;
4664 }
4665 module_init(hns3_init_module);
4666
4667 /* hns3_exit_module - Driver exit cleanup routine
4668  * hns3_exit_module is called just before the driver is removed
4669  * from memory.
4670  */
4671 static void __exit hns3_exit_module(void)
4672 {
4673         pci_unregister_driver(&hns3_driver);
4674         hnae3_unregister_client(&client);
4675         hns3_dbg_unregister_debugfs();
4676 }
4677 module_exit(hns3_exit_module);
4678
4679 MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
4680 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
4681 MODULE_LICENSE("GPL");
4682 MODULE_ALIAS("pci:hns-nic");