drivers/net/ethernet/freescale/enetc/enetc.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /* Copyright 2017-2019 NXP */
   3
   4 #include "enetc.h"
   5 #include <linux/bpf_trace.h>
   6 #include <linux/tcp.h>
   7 #include <linux/udp.h>
   8 #include <linux/vmalloc.h>
   9 #include <linux/ptp_classify.h>
  10 #include <net/ip6_checksum.h>
  11 #include <net/pkt_sched.h>
  12 #include <net/tso.h>
  13
  14 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
  15 {
  16         int num_tx_rings = priv->num_tx_rings;
  17         int i;
  18
  19         for (i = 0; i < priv->num_rx_rings; i++)
  20                 if (priv->rx_ring[i]->xdp.prog)
  21                         return num_tx_rings - num_possible_cpus();
  22
  23         return num_tx_rings;
  24 }
  25
  26 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
  27                                                         struct enetc_bdr *tx_ring)
  28 {
  29         int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
  30
  31         return priv->rx_ring[index];
  32 }
  33
  34 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
  35 {
  36         if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
  37                 return NULL;
  38
  39         return tx_swbd->skb;
  40 }
  41
  42 static struct xdp_frame *
  43 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
  44 {
  45         if (tx_swbd->is_xdp_redirect)
  46                 return tx_swbd->xdp_frame;
  47
  48         return NULL;
  49 }
  50
  51 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
  52                                 struct enetc_tx_swbd *tx_swbd)
  53 {
  54         /* For XDP_TX, pages come from RX, whereas for the other contexts where
  55          * we have is_dma_page_set, those come from skb_frag_dma_map. We need
  56          * to match the DMA mapping length, so we need to differentiate those.
  57          */
  58         if (tx_swbd->is_dma_page)
  59                 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
  60                                tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
  61                                tx_swbd->dir);
  62         else
  63                 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
  64                                  tx_swbd->len, tx_swbd->dir);
  65         tx_swbd->dma = 0;
  66 }
  67
  68 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
  69                                 struct enetc_tx_swbd *tx_swbd)
  70 {
  71         struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
  72         struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
  73
  74         if (tx_swbd->dma)
  75                 enetc_unmap_tx_buff(tx_ring, tx_swbd);
  76
  77         if (xdp_frame) {
  78                 xdp_return_frame(tx_swbd->xdp_frame);
  79                 tx_swbd->xdp_frame = NULL;
  80         } else if (skb) {
  81                 dev_kfree_skb_any(skb);
  82                 tx_swbd->skb = NULL;
  83         }
  84 }
  85
  86 /* Let H/W know BD ring has been updated */
  87 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
  88 {
  89         /* includes wmb() */
  90         enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
  91 }
  92
  93 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
  94                            u8 *msgtype, u8 *twostep,
  95                            u16 *correction_offset, u16 *body_offset)
  96 {
  97         unsigned int ptp_class;
  98         struct ptp_header *hdr;
  99         unsigned int type;
 100         u8 *base;
 101
 102         ptp_class = ptp_classify_raw(skb);
 103         if (ptp_class == PTP_CLASS_NONE)
 104                 return -EINVAL;
 105
 106         hdr = ptp_parse_header(skb, ptp_class);
 107         if (!hdr)
 108                 return -EINVAL;
 109
 110         type = ptp_class & PTP_CLASS_PMASK;
 111         if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
 112                 *udp = 1;
 113         else
 114                 *udp = 0;
 115
 116         *msgtype = ptp_get_msgtype(hdr, ptp_class);
 117         *twostep = hdr->flag_field[0] & 0x2;
 118
 119         base = skb_mac_header(skb);
 120         *correction_offset = (u8 *)&hdr->correction - base;
 121         *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
 122
 123         return 0;
 124 }
 125
 126 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
 127 {
 128         bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
 129         struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
 130         struct enetc_hw *hw = &priv->si->hw;
 131         struct enetc_tx_swbd *tx_swbd;
 132         int len = skb_headlen(skb);
 133         union enetc_tx_bd temp_bd;
 134         u8 msgtype, twostep, udp;
 135         union enetc_tx_bd *txbd;
 136         u16 offset1, offset2;
 137         int i, count = 0;
 138         skb_frag_t *frag;
 139         unsigned int f;
 140         dma_addr_t dma;
 141         u8 flags = 0;
 142
 143         i = tx_ring->next_to_use;
 144         txbd = ENETC_TXBD(*tx_ring, i);
 145         prefetchw(txbd);
 146
 147         dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
 148         if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
 149                 goto dma_err;
 150
 151         temp_bd.addr = cpu_to_le64(dma);
 152         temp_bd.buf_len = cpu_to_le16(len);
 153         temp_bd.lstatus = 0;
 154
 155         tx_swbd = &tx_ring->tx_swbd[i];
 156         tx_swbd->dma = dma;
 157         tx_swbd->len = len;
 158         tx_swbd->is_dma_page = 0;
 159         tx_swbd->dir = DMA_TO_DEVICE;
 160         count++;
 161
 162         do_vlan = skb_vlan_tag_present(skb);
 163         if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 164                 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
 165                                     &offset2) ||
 166                     msgtype != PTP_MSGTYPE_SYNC || twostep)
 167                         WARN_ONCE(1, "Bad packet for one-step timestamping\n");
 168                 else
 169                         do_onestep_tstamp = true;
 170         } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
 171                 do_twostep_tstamp = true;
 172         }
 173
 174         tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
 175         tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
 176         tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
 177
 178         if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
 179                 flags |= ENETC_TXBD_FLAGS_EX;
 180
 181         if (tx_ring->tsd_enable)
 182                 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
 183
 184         /* first BD needs frm_len and offload flags set */
 185         temp_bd.frm_len = cpu_to_le16(skb->len);
 186         temp_bd.flags = flags;
 187
 188         if (flags & ENETC_TXBD_FLAGS_TSE)
 189                 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
 190                                                           flags);
 191
 192         if (flags & ENETC_TXBD_FLAGS_EX) {
 193                 u8 e_flags = 0;
 194                 *txbd = temp_bd;
 195                 enetc_clear_tx_bd(&temp_bd);
 196
 197                 /* add extension BD for VLAN and/or timestamping */
 198                 flags = 0;
 199                 tx_swbd++;
 200                 txbd++;
 201                 i++;
 202                 if (unlikely(i == tx_ring->bd_count)) {
 203                         i = 0;
 204                         tx_swbd = tx_ring->tx_swbd;
 205                         txbd = ENETC_TXBD(*tx_ring, 0);
 206                 }
 207                 prefetchw(txbd);
 208
 209                 if (do_vlan) {
 210                         temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
 211                         temp_bd.ext.tpid = 0; /* < C-TAG */
 212                         e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
 213                 }
 214
 215                 if (do_onestep_tstamp) {
 216                         u32 lo, hi, val;
 217                         u64 sec, nsec;
 218                         u8 *data;
 219
 220                         lo = enetc_rd_hot(hw, ENETC_SICTR0);
 221                         hi = enetc_rd_hot(hw, ENETC_SICTR1);
 222                         sec = (u64)hi << 32 | lo;
 223                         nsec = do_div(sec, 1000000000);
 224
 225                         /* Configure extension BD */
 226                         temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
 227                         e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
 228
 229                         /* Update originTimestamp field of Sync packet
 230                          * - 48 bits seconds field
 231                          * - 32 bits nanseconds field
 232                          */
 233                         data = skb_mac_header(skb);
 234                         *(__be16 *)(data + offset2) =
 235                                 htons((sec >> 32) & 0xffff);
 236                         *(__be32 *)(data + offset2 + 2) =
 237                                 htonl(sec & 0xffffffff);
 238                         *(__be32 *)(data + offset2 + 6) = htonl(nsec);
 239
 240                         /* Configure single-step register */
 241                         val = ENETC_PM0_SINGLE_STEP_EN;
 242                         val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
 243                         if (udp)
 244                                 val |= ENETC_PM0_SINGLE_STEP_CH;
 245
 246                         enetc_port_wr(hw, ENETC_PM0_SINGLE_STEP, val);
 247                         enetc_port_wr(hw, ENETC_PM1_SINGLE_STEP, val);
 248                 } else if (do_twostep_tstamp) {
 249                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 250                         e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
 251                 }
 252
 253                 temp_bd.ext.e_flags = e_flags;
 254                 count++;
 255         }
 256
 257         frag = &skb_shinfo(skb)->frags[0];
 258         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
 259                 len = skb_frag_size(frag);
 260                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
 261                                        DMA_TO_DEVICE);
 262                 if (dma_mapping_error(tx_ring->dev, dma))
 263                         goto dma_err;
 264
 265                 *txbd = temp_bd;
 266                 enetc_clear_tx_bd(&temp_bd);
 267
 268                 flags = 0;
 269                 tx_swbd++;
 270                 txbd++;
 271                 i++;
 272                 if (unlikely(i == tx_ring->bd_count)) {
 273                         i = 0;
 274                         tx_swbd = tx_ring->tx_swbd;
 275                         txbd = ENETC_TXBD(*tx_ring, 0);
 276                 }
 277                 prefetchw(txbd);
 278
 279                 temp_bd.addr = cpu_to_le64(dma);
 280                 temp_bd.buf_len = cpu_to_le16(len);
 281
 282                 tx_swbd->dma = dma;
 283                 tx_swbd->len = len;
 284                 tx_swbd->is_dma_page = 1;
 285                 tx_swbd->dir = DMA_TO_DEVICE;
 286                 count++;
 287         }
 288
 289         /* last BD needs 'F' bit set */
 290         flags |= ENETC_TXBD_FLAGS_F;
 291         temp_bd.flags = flags;
 292         *txbd = temp_bd;
 293
 294         tx_ring->tx_swbd[i].is_eof = true;
 295         tx_ring->tx_swbd[i].skb = skb;
 296
 297         enetc_bdr_idx_inc(tx_ring, &i);
 298         tx_ring->next_to_use = i;
 299
 300         skb_tx_timestamp(skb);
 301
 302         enetc_update_tx_ring_tail(tx_ring);
 303
 304         return count;
 305
 306 dma_err:
 307         dev_err(tx_ring->dev, "DMA map error");
 308
 309         do {
 310                 tx_swbd = &tx_ring->tx_swbd[i];
 311                 enetc_free_tx_frame(tx_ring, tx_swbd);
 312                 if (i == 0)
 313                         i = tx_ring->bd_count;
 314                 i--;
 315         } while (count--);
 316
 317         return 0;
 318 }
 319
 320 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
 321                                  struct enetc_tx_swbd *tx_swbd,
 322                                  union enetc_tx_bd *txbd, int *i, int hdr_len,
 323                                  int data_len)
 324 {
 325         union enetc_tx_bd txbd_tmp;
 326         u8 flags = 0, e_flags = 0;
 327         dma_addr_t addr;
 328
 329         enetc_clear_tx_bd(&txbd_tmp);
 330         addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
 331
 332         if (skb_vlan_tag_present(skb))
 333                 flags |= ENETC_TXBD_FLAGS_EX;
 334
 335         txbd_tmp.addr = cpu_to_le64(addr);
 336         txbd_tmp.buf_len = cpu_to_le16(hdr_len);
 337
 338         /* first BD needs frm_len and offload flags set */
 339         txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
 340         txbd_tmp.flags = flags;
 341
 342         /* For the TSO header we do not set the dma address since we do not
 343          * want it unmapped when we do cleanup. We still set len so that we
 344          * count the bytes sent.
 345          */
 346         tx_swbd->len = hdr_len;
 347         tx_swbd->do_twostep_tstamp = false;
 348         tx_swbd->check_wb = false;
 349
 350         /* Actually write the header in the BD */
 351         *txbd = txbd_tmp;
 352
 353         /* Add extension BD for VLAN */
 354         if (flags & ENETC_TXBD_FLAGS_EX) {
 355                 /* Get the next BD */
 356                 enetc_bdr_idx_inc(tx_ring, i);
 357                 txbd = ENETC_TXBD(*tx_ring, *i);
 358                 tx_swbd = &tx_ring->tx_swbd[*i];
 359                 prefetchw(txbd);
 360
 361                 /* Setup the VLAN fields */
 362                 enetc_clear_tx_bd(&txbd_tmp);
 363                 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
 364                 txbd_tmp.ext.tpid = 0; /* < C-TAG */
 365                 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
 366
 367                 /* Write the BD */
 368                 txbd_tmp.ext.e_flags = e_flags;
 369                 *txbd = txbd_tmp;
 370         }
 371 }
 372
 373 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
 374                                  struct enetc_tx_swbd *tx_swbd,
 375                                  union enetc_tx_bd *txbd, char *data,
 376                                  int size, bool last_bd)
 377 {
 378         union enetc_tx_bd txbd_tmp;
 379         dma_addr_t addr;
 380         u8 flags = 0;
 381
 382         enetc_clear_tx_bd(&txbd_tmp);
 383
 384         addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
 385         if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
 386                 netdev_err(tx_ring->ndev, "DMA map error\n");
 387                 return -ENOMEM;
 388         }
 389
 390         if (last_bd) {
 391                 flags |= ENETC_TXBD_FLAGS_F;
 392                 tx_swbd->is_eof = 1;
 393         }
 394
 395         txbd_tmp.addr = cpu_to_le64(addr);
 396         txbd_tmp.buf_len = cpu_to_le16(size);
 397         txbd_tmp.flags = flags;
 398
 399         tx_swbd->dma = addr;
 400         tx_swbd->len = size;
 401         tx_swbd->dir = DMA_TO_DEVICE;
 402
 403         *txbd = txbd_tmp;
 404
 405         return 0;
 406 }
 407
 408 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
 409                                  char *hdr, int hdr_len, int *l4_hdr_len)
 410 {
 411         char *l4_hdr = hdr + skb_transport_offset(skb);
 412         int mac_hdr_len = skb_network_offset(skb);
 413
 414         if (tso->tlen != sizeof(struct udphdr)) {
 415                 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
 416
 417                 tcph->check = 0;
 418         } else {
 419                 struct udphdr *udph = (struct udphdr *)(l4_hdr);
 420
 421                 udph->check = 0;
 422         }
 423
 424         /* Compute the IP checksum. This is necessary since tso_build_hdr()
 425          * already incremented the IP ID field.
 426          */
 427         if (!tso->ipv6) {
 428                 struct iphdr *iph = (void *)(hdr + mac_hdr_len);
 429
 430                 iph->check = 0;
 431                 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
 432         }
 433
 434         /* Compute the checksum over the L4 header. */
 435         *l4_hdr_len = hdr_len - skb_transport_offset(skb);
 436         return csum_partial(l4_hdr, *l4_hdr_len, 0);
 437 }
 438
 439 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
 440                                     struct sk_buff *skb, char *hdr, int len,
 441                                     __wsum sum)
 442 {
 443         char *l4_hdr = hdr + skb_transport_offset(skb);
 444         __sum16 csum_final;
 445
 446         /* Complete the L4 checksum by appending the pseudo-header to the
 447          * already computed checksum.
 448          */
 449         if (!tso->ipv6)
 450                 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
 451                                                ip_hdr(skb)->daddr,
 452                                                len, ip_hdr(skb)->protocol, sum);
 453         else
 454                 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
 455                                              &ipv6_hdr(skb)->daddr,
 456                                              len, ipv6_hdr(skb)->nexthdr, sum);
 457
 458         if (tso->tlen != sizeof(struct udphdr)) {
 459                 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
 460
 461                 tcph->check = csum_final;
 462         } else {
 463                 struct udphdr *udph = (struct udphdr *)(l4_hdr);
 464
 465                 udph->check = csum_final;
 466         }
 467 }
 468
 469 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
 470 {
 471         int hdr_len, total_len, data_len;
 472         struct enetc_tx_swbd *tx_swbd;
 473         union enetc_tx_bd *txbd;
 474         struct tso_t tso;
 475         __wsum csum, csum2;
 476         int count = 0, pos;
 477         int err, i, bd_data_num;
 478
 479         /* Initialize the TSO handler, and prepare the first payload */
 480         hdr_len = tso_start(skb, &tso);
 481         total_len = skb->len - hdr_len;
 482         i = tx_ring->next_to_use;
 483
 484         while (total_len > 0) {
 485                 char *hdr;
 486
 487                 /* Get the BD */
 488                 txbd = ENETC_TXBD(*tx_ring, i);
 489                 tx_swbd = &tx_ring->tx_swbd[i];
 490                 prefetchw(txbd);
 491
 492                 /* Determine the length of this packet */
 493                 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
 494                 total_len -= data_len;
 495
 496                 /* prepare packet headers: MAC + IP + TCP */
 497                 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
 498                 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
 499
 500                 /* compute the csum over the L4 header */
 501                 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
 502                 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);
 503                 bd_data_num = 0;
 504                 count++;
 505
 506                 while (data_len > 0) {
 507                         int size;
 508
 509                         size = min_t(int, tso.size, data_len);
 510
 511                         /* Advance the index in the BDR */
 512                         enetc_bdr_idx_inc(tx_ring, &i);
 513                         txbd = ENETC_TXBD(*tx_ring, i);
 514                         tx_swbd = &tx_ring->tx_swbd[i];
 515                         prefetchw(txbd);
 516
 517                         /* Compute the checksum over this segment of data and
 518                          * add it to the csum already computed (over the L4
 519                          * header and possible other data segments).
 520                          */
 521                         csum2 = csum_partial(tso.data, size, 0);
 522                         csum = csum_block_add(csum, csum2, pos);
 523                         pos += size;
 524
 525                         err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
 526                                                     tso.data, size,
 527                                                     size == data_len);
 528                         if (err)
 529                                 goto err_map_data;
 530
 531                         data_len -= size;
 532                         count++;
 533                         bd_data_num++;
 534                         tso_build_data(skb, &tso, size);
 535
 536                         if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))
 537                                 goto err_chained_bd;
 538                 }
 539
 540                 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
 541
 542                 if (total_len == 0)
 543                         tx_swbd->skb = skb;
 544
 545                 /* Go to the next BD */
 546                 enetc_bdr_idx_inc(tx_ring, &i);
 547         }
 548
 549         tx_ring->next_to_use = i;
 550         enetc_update_tx_ring_tail(tx_ring);
 551
 552         return count;
 553
 554 err_map_data:
 555         dev_err(tx_ring->dev, "DMA map error");
 556
 557 err_chained_bd:
 558         do {
 559                 tx_swbd = &tx_ring->tx_swbd[i];
 560                 enetc_free_tx_frame(tx_ring, tx_swbd);
 561                 if (i == 0)
 562                         i = tx_ring->bd_count;
 563                 i--;
 564         } while (count--);
 565
 566         return 0;
 567 }
 568
 569 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
 570                                     struct net_device *ndev)
 571 {
 572         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 573         struct enetc_bdr *tx_ring;
 574         int count, err;
 575
 576         /* Queue one-step Sync packet if already locked */
 577         if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 578                 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
 579                                           &priv->flags)) {
 580                         skb_queue_tail(&priv->tx_skbs, skb);
 581                         return NETDEV_TX_OK;
 582                 }
 583         }
 584
 585         tx_ring = priv->tx_ring[skb->queue_mapping];
 586
 587         if (skb_is_gso(skb)) {
 588                 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
 589                         netif_stop_subqueue(ndev, tx_ring->index);
 590                         return NETDEV_TX_BUSY;
 591                 }
 592
 593                 enetc_lock_mdio();
 594                 count = enetc_map_tx_tso_buffs(tx_ring, skb);
 595                 enetc_unlock_mdio();
 596         } else {
 597                 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
 598                         if (unlikely(skb_linearize(skb)))
 599                                 goto drop_packet_err;
 600
 601                 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
 602                 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
 603                         netif_stop_subqueue(ndev, tx_ring->index);
 604                         return NETDEV_TX_BUSY;
 605                 }
 606
 607                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
 608                         err = skb_checksum_help(skb);
 609                         if (err)
 610                                 goto drop_packet_err;
 611                 }
 612                 enetc_lock_mdio();
 613                 count = enetc_map_tx_buffs(tx_ring, skb);
 614                 enetc_unlock_mdio();
 615         }
 616
 617         if (unlikely(!count))
 618                 goto drop_packet_err;
 619
 620         if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
 621                 netif_stop_subqueue(ndev, tx_ring->index);
 622
 623         return NETDEV_TX_OK;
 624
 625 drop_packet_err:
 626         dev_kfree_skb_any(skb);
 627         return NETDEV_TX_OK;
 628 }
 629
 630 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
 631 {
 632         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 633         u8 udp, msgtype, twostep;
 634         u16 offset1, offset2;
 635
 636         /* Mark tx timestamp type on skb->cb[0] if requires */
 637         if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
 638             (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
 639                 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
 640         } else {
 641                 skb->cb[0] = 0;
 642         }
 643
 644         /* Fall back to two-step timestamp if not one-step Sync packet */
 645         if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 646                 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
 647                                     &offset1, &offset2) ||
 648                     msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
 649                         skb->cb[0] = ENETC_F_TX_TSTAMP;
 650         }
 651
 652         return enetc_start_xmit(skb, ndev);
 653 }
 654
 655 static irqreturn_t enetc_msix(int irq, void *data)
 656 {
 657         struct enetc_int_vector *v = data;
 658         int i;
 659
 660         enetc_lock_mdio();
 661
 662         /* disable interrupts */
 663         enetc_wr_reg_hot(v->rbier, 0);
 664         enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
 665
 666         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
 667                 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
 668
 669         enetc_unlock_mdio();
 670
 671         napi_schedule(&v->napi);
 672
 673         return IRQ_HANDLED;
 674 }
 675
 676 static void enetc_rx_dim_work(struct work_struct *w)
 677 {
 678         struct dim *dim = container_of(w, struct dim, work);
 679         struct dim_cq_moder moder =
 680                 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
 681         struct enetc_int_vector *v =
 682                 container_of(dim, struct enetc_int_vector, rx_dim);
 683
 684         v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
 685         dim->state = DIM_START_MEASURE;
 686 }
 687
 688 static void enetc_rx_net_dim(struct enetc_int_vector *v)
 689 {
 690         struct dim_sample dim_sample = {};
 691
 692         v->comp_cnt++;
 693
 694         if (!v->rx_napi_work)
 695                 return;
 696
 697         dim_update_sample(v->comp_cnt,
 698                           v->rx_ring.stats.packets,
 699                           v->rx_ring.stats.bytes,
 700                           &dim_sample);
 701         net_dim(&v->rx_dim, dim_sample);
 702 }
 703
 704 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
 705 {
 706         int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
 707
 708         return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
 709 }
 710
 711 static bool enetc_page_reusable(struct page *page)
 712 {
 713         return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
 714 }
 715
 716 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
 717                              struct enetc_rx_swbd *old)
 718 {
 719         struct enetc_rx_swbd *new;
 720
 721         new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
 722
 723         /* next buf that may reuse a page */
 724         enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
 725
 726         /* copy page reference */
 727         *new = *old;
 728 }
 729
 730 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
 731                                 u64 *tstamp)
 732 {
 733         u32 lo, hi, tstamp_lo;
 734
 735         lo = enetc_rd_hot(hw, ENETC_SICTR0);
 736         hi = enetc_rd_hot(hw, ENETC_SICTR1);
 737         tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
 738         if (lo <= tstamp_lo)
 739                 hi -= 1;
 740         *tstamp = (u64)hi << 32 | tstamp_lo;
 741 }
 742
 743 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
 744 {
 745         struct skb_shared_hwtstamps shhwtstamps;
 746
 747         if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
 748                 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
 749                 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
 750                 skb_txtime_consumed(skb);
 751                 skb_tstamp_tx(skb, &shhwtstamps);
 752         }
 753 }
 754
 755 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
 756                                       struct enetc_tx_swbd *tx_swbd)
 757 {
 758         struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
 759         struct enetc_rx_swbd rx_swbd = {
 760                 .dma = tx_swbd->dma,
 761                 .page = tx_swbd->page,
 762                 .page_offset = tx_swbd->page_offset,
 763                 .dir = tx_swbd->dir,
 764                 .len = tx_swbd->len,
 765         };
 766         struct enetc_bdr *rx_ring;
 767
 768         rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
 769
 770         if (likely(enetc_swbd_unused(rx_ring))) {
 771                 enetc_reuse_page(rx_ring, &rx_swbd);
 772
 773                 /* sync for use by the device */
 774                 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
 775                                                  rx_swbd.page_offset,
 776                                                  ENETC_RXB_DMA_SIZE_XDP,
 777                                                  rx_swbd.dir);
 778
 779                 rx_ring->stats.recycles++;
 780         } else {
 781                 /* RX ring is already full, we need to unmap and free the
 782                  * page, since there's nothing useful we can do with it.
 783                  */
 784                 rx_ring->stats.recycle_failures++;
 785
 786                 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
 787                                rx_swbd.dir);
 788                 __free_page(rx_swbd.page);
 789         }
 790
 791         rx_ring->xdp.xdp_tx_in_flight--;
 792 }
 793
 794 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
 795 {
 796         int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
 797         struct net_device *ndev = tx_ring->ndev;
 798         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 799         struct enetc_tx_swbd *tx_swbd;
 800         int i, bds_to_clean;
 801         bool do_twostep_tstamp;
 802         u64 tstamp = 0;
 803
 804         i = tx_ring->next_to_clean;
 805         tx_swbd = &tx_ring->tx_swbd[i];
 806
 807         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 808
 809         do_twostep_tstamp = false;
 810
 811         while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
 812                 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
 813                 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
 814                 bool is_eof = tx_swbd->is_eof;
 815
 816                 if (unlikely(tx_swbd->check_wb)) {
 817                         union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
 818
 819                         if (txbd->flags & ENETC_TXBD_FLAGS_W &&
 820                             tx_swbd->do_twostep_tstamp) {
 821                                 enetc_get_tx_tstamp(&priv->si->hw, txbd,
 822                                                     &tstamp);
 823                                 do_twostep_tstamp = true;
 824                         }
 825
 826                         if (tx_swbd->qbv_en &&
 827                             txbd->wb.status & ENETC_TXBD_STATS_WIN)
 828                                 tx_win_drop++;
 829                 }
 830
 831                 if (tx_swbd->is_xdp_tx)
 832                         enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
 833                 else if (likely(tx_swbd->dma))
 834                         enetc_unmap_tx_buff(tx_ring, tx_swbd);
 835
 836                 if (xdp_frame) {
 837                         xdp_return_frame(xdp_frame);
 838                 } else if (skb) {
 839                         if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
 840                                 /* Start work to release lock for next one-step
 841                                  * timestamping packet. And send one skb in
 842                                  * tx_skbs queue if has.
 843                                  */
 844                                 schedule_work(&priv->tx_onestep_tstamp);
 845                         } else if (unlikely(do_twostep_tstamp)) {
 846                                 enetc_tstamp_tx(skb, tstamp);
 847                                 do_twostep_tstamp = false;
 848                         }
 849                         napi_consume_skb(skb, napi_budget);
 850                 }
 851
 852                 tx_byte_cnt += tx_swbd->len;
 853                 /* Scrub the swbd here so we don't have to do that
 854                  * when we reuse it during xmit
 855                  */
 856                 memset(tx_swbd, 0, sizeof(*tx_swbd));
 857
 858                 bds_to_clean--;
 859                 tx_swbd++;
 860                 i++;
 861                 if (unlikely(i == tx_ring->bd_count)) {
 862                         i = 0;
 863                         tx_swbd = tx_ring->tx_swbd;
 864                 }
 865
 866                 /* BD iteration loop end */
 867                 if (is_eof) {
 868                         tx_frm_cnt++;
 869                         /* re-arm interrupt source */
 870                         enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
 871                                          BIT(16 + tx_ring->index));
 872                 }
 873
 874                 if (unlikely(!bds_to_clean))
 875                         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 876         }
 877
 878         tx_ring->next_to_clean = i;
 879         tx_ring->stats.packets += tx_frm_cnt;
 880         tx_ring->stats.bytes += tx_byte_cnt;
 881         tx_ring->stats.win_drop += tx_win_drop;
 882
 883         if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
 884                      __netif_subqueue_stopped(ndev, tx_ring->index) &&
 885                      (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
 886                 netif_wake_subqueue(ndev, tx_ring->index);
 887         }
 888
 889         return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
 890 }
 891
 892 static bool enetc_new_page(struct enetc_bdr *rx_ring,
 893                            struct enetc_rx_swbd *rx_swbd)
 894 {
 895         bool xdp = !!(rx_ring->xdp.prog);
 896         struct page *page;
 897         dma_addr_t addr;
 898
 899         page = dev_alloc_page();
 900         if (unlikely(!page))
 901                 return false;
 902
 903         /* For XDP_TX, we forgo dma_unmap -> dma_map */
 904         rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 905
 906         addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
 907         if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
 908                 __free_page(page);
 909
 910                 return false;
 911         }
 912
 913         rx_swbd->dma = addr;
 914         rx_swbd->page = page;
 915         rx_swbd->page_offset = rx_ring->buffer_offset;
 916
 917         return true;
 918 }
 919
 920 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
 921 {
 922         struct enetc_rx_swbd *rx_swbd;
 923         union enetc_rx_bd *rxbd;
 924         int i, j;
 925
 926         i = rx_ring->next_to_use;
 927         rx_swbd = &rx_ring->rx_swbd[i];
 928         rxbd = enetc_rxbd(rx_ring, i);
 929
 930         for (j = 0; j < buff_cnt; j++) {
 931                 /* try reuse page */
 932                 if (unlikely(!rx_swbd->page)) {
 933                         if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
 934                                 rx_ring->stats.rx_alloc_errs++;
 935                                 break;
 936                         }
 937                 }
 938
 939                 /* update RxBD */
 940                 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
 941                                            rx_swbd->page_offset);
 942                 /* clear 'R" as well */
 943                 rxbd->r.lstatus = 0;
 944
 945                 enetc_rxbd_next(rx_ring, &rxbd, &i);
 946                 rx_swbd = &rx_ring->rx_swbd[i];
 947         }
 948
 949         if (likely(j)) {
 950                 rx_ring->next_to_alloc = i; /* keep track from page reuse */
 951                 rx_ring->next_to_use = i;
 952
 953                 /* update ENETC's consumer index */
 954                 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
 955         }
 956
 957         return j;
 958 }
 959
 960 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 961 static void enetc_get_rx_tstamp(struct net_device *ndev,
 962                                 union enetc_rx_bd *rxbd,
 963                                 struct sk_buff *skb)
 964 {
 965         struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
 966         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 967         struct enetc_hw *hw = &priv->si->hw;
 968         u32 lo, hi, tstamp_lo;
 969         u64 tstamp;
 970
 971         if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
 972                 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
 973                 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
 974                 rxbd = enetc_rxbd_ext(rxbd);
 975                 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
 976                 if (lo <= tstamp_lo)
 977                         hi -= 1;
 978
 979                 tstamp = (u64)hi << 32 | tstamp_lo;
 980                 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
 981                 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
 982         }
 983 }
 984 #endif
 985
 986 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
 987                                union enetc_rx_bd *rxbd, struct sk_buff *skb)
 988 {
 989         struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
 990
 991         /* TODO: hashing */
 992         if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
 993                 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
 994
 995                 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
 996                 skb->ip_summed = CHECKSUM_COMPLETE;
 997         }
 998
 999         if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1000                 __be16 tpid = 0;
1001
1002                 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1003                 case 0:
1004                         tpid = htons(ETH_P_8021Q);
1005                         break;
1006                 case 1:
1007                         tpid = htons(ETH_P_8021AD);
1008                         break;
1009                 case 2:
1010                         tpid = htons(enetc_port_rd(&priv->si->hw,
1011                                                    ENETC_PCVLANR1));
1012                         break;
1013                 case 3:
1014                         tpid = htons(enetc_port_rd(&priv->si->hw,
1015                                                    ENETC_PCVLANR2));
1016                         break;
1017                 default:
1018                         break;
1019                 }
1020
1021                 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1022         }
1023
1024 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1025         if (priv->active_offloads & ENETC_F_RX_TSTAMP)
1026                 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1027 #endif
1028 }
1029
1030 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1031  * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1032  * mapped buffers.
1033  */
1034 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1035                                                int i, u16 size)
1036 {
1037         struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1038
1039         dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1040                                       rx_swbd->page_offset,
1041                                       size, rx_swbd->dir);
1042         return rx_swbd;
1043 }
1044
1045 /* Reuse the current page without performing half-page buffer flipping */
1046 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1047                               struct enetc_rx_swbd *rx_swbd)
1048 {
1049         size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1050
1051         enetc_reuse_page(rx_ring, rx_swbd);
1052
1053         dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1054                                          rx_swbd->page_offset,
1055                                          buffer_size, rx_swbd->dir);
1056
1057         rx_swbd->page = NULL;
1058 }
1059
1060 /* Reuse the current page by performing half-page buffer flipping */
1061 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1062                                struct enetc_rx_swbd *rx_swbd)
1063 {
1064         if (likely(enetc_page_reusable(rx_swbd->page))) {
1065                 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1066                 page_ref_inc(rx_swbd->page);
1067
1068                 enetc_put_rx_buff(rx_ring, rx_swbd);
1069         } else {
1070                 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1071                                rx_swbd->dir);
1072                 rx_swbd->page = NULL;
1073         }
1074 }
1075
1076 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1077                                                 int i, u16 size)
1078 {
1079         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1080         struct sk_buff *skb;
1081         void *ba;
1082
1083         ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1084         skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1085         if (unlikely(!skb)) {
1086                 rx_ring->stats.rx_alloc_errs++;
1087                 return NULL;
1088         }
1089
1090         skb_reserve(skb, rx_ring->buffer_offset);
1091         __skb_put(skb, size);
1092
1093         enetc_flip_rx_buff(rx_ring, rx_swbd);
1094
1095         return skb;
1096 }
1097
1098 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1099                                      u16 size, struct sk_buff *skb)
1100 {
1101         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1102
1103         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1104                         rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1105
1106         enetc_flip_rx_buff(rx_ring, rx_swbd);
1107 }
1108
1109 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1110                                               u32 bd_status,
1111                                               union enetc_rx_bd **rxbd, int *i)
1112 {
1113         if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1114                 return false;
1115
1116         enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1117         enetc_rxbd_next(rx_ring, rxbd, i);
1118
1119         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1120                 dma_rmb();
1121                 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1122
1123                 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1124                 enetc_rxbd_next(rx_ring, rxbd, i);
1125         }
1126
1127         rx_ring->ndev->stats.rx_dropped++;
1128         rx_ring->ndev->stats.rx_errors++;
1129
1130         return true;
1131 }
1132
1133 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1134                                        u32 bd_status, union enetc_rx_bd **rxbd,
1135                                        int *i, int *cleaned_cnt, int buffer_size)
1136 {
1137         struct sk_buff *skb;
1138         u16 size;
1139
1140         size = le16_to_cpu((*rxbd)->r.buf_len);
1141         skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1142         if (!skb)
1143                 return NULL;
1144
1145         enetc_get_offloads(rx_ring, *rxbd, skb);
1146
1147         (*cleaned_cnt)++;
1148
1149         enetc_rxbd_next(rx_ring, rxbd, i);
1150
1151         /* not last BD in frame? */
1152         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1153                 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1154                 size = buffer_size;
1155
1156                 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1157                         dma_rmb();
1158                         size = le16_to_cpu((*rxbd)->r.buf_len);
1159                 }
1160
1161                 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1162
1163                 (*cleaned_cnt)++;
1164
1165                 enetc_rxbd_next(rx_ring, rxbd, i);
1166         }
1167
1168         skb_record_rx_queue(skb, rx_ring->index);
1169         skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1170
1171         return skb;
1172 }
1173
1174 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1175
1176 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1177                                struct napi_struct *napi, int work_limit)
1178 {
1179         int rx_frm_cnt = 0, rx_byte_cnt = 0;
1180         int cleaned_cnt, i;
1181
1182         cleaned_cnt = enetc_bd_unused(rx_ring);
1183         /* next descriptor to process */
1184         i = rx_ring->next_to_clean;
1185
1186         while (likely(rx_frm_cnt < work_limit)) {
1187                 union enetc_rx_bd *rxbd;
1188                 struct sk_buff *skb;
1189                 u32 bd_status;
1190
1191                 if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1192                         cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1193                                                             cleaned_cnt);
1194
1195                 rxbd = enetc_rxbd(rx_ring, i);
1196                 bd_status = le32_to_cpu(rxbd->r.lstatus);
1197                 if (!bd_status)
1198                         break;
1199
1200                 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1201                 dma_rmb(); /* for reading other rxbd fields */
1202
1203                 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1204                                                       &rxbd, &i))
1205                         break;
1206
1207                 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1208                                       &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1209                 if (!skb)
1210                         break;
1211
1212                 /* When set, the outer VLAN header is extracted and reported
1213                  * in the receive buffer descriptor. So rx_byte_cnt should
1214                  * add the length of the extracted VLAN header.
1215                  */
1216                 if (bd_status & ENETC_RXBD_FLAG_VLAN)
1217                         rx_byte_cnt += VLAN_HLEN;
1218                 rx_byte_cnt += skb->len + ETH_HLEN;
1219                 rx_frm_cnt++;
1220
1221                 napi_gro_receive(napi, skb);
1222         }
1223
1224         rx_ring->next_to_clean = i;
1225
1226         rx_ring->stats.packets += rx_frm_cnt;
1227         rx_ring->stats.bytes += rx_byte_cnt;
1228
1229         return rx_frm_cnt;
1230 }
1231
1232 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1233                                   struct enetc_tx_swbd *tx_swbd,
1234                                   int frm_len)
1235 {
1236         union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1237
1238         prefetchw(txbd);
1239
1240         enetc_clear_tx_bd(txbd);
1241         txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1242         txbd->buf_len = cpu_to_le16(tx_swbd->len);
1243         txbd->frm_len = cpu_to_le16(frm_len);
1244
1245         memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1246 }
1247
1248 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1249  * descriptors.
1250  */
1251 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1252                          struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1253 {
1254         struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1255         int i, k, frm_len = tmp_tx_swbd->len;
1256
1257         if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1258                 return false;
1259
1260         while (unlikely(!tmp_tx_swbd->is_eof)) {
1261                 tmp_tx_swbd++;
1262                 frm_len += tmp_tx_swbd->len;
1263         }
1264
1265         i = tx_ring->next_to_use;
1266
1267         for (k = 0; k < num_tx_swbd; k++) {
1268                 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1269
1270                 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1271
1272                 /* last BD needs 'F' bit set */
1273                 if (xdp_tx_swbd->is_eof) {
1274                         union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1275
1276                         txbd->flags = ENETC_TXBD_FLAGS_F;
1277                 }
1278
1279                 enetc_bdr_idx_inc(tx_ring, &i);
1280         }
1281
1282         tx_ring->next_to_use = i;
1283
1284         return true;
1285 }
1286
1287 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1288                                           struct enetc_tx_swbd *xdp_tx_arr,
1289                                           struct xdp_frame *xdp_frame)
1290 {
1291         struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1292         struct skb_shared_info *shinfo;
1293         void *data = xdp_frame->data;
1294         int len = xdp_frame->len;
1295         skb_frag_t *frag;
1296         dma_addr_t dma;
1297         unsigned int f;
1298         int n = 0;
1299
1300         dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1301         if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1302                 netdev_err(tx_ring->ndev, "DMA map error\n");
1303                 return -1;
1304         }
1305
1306         xdp_tx_swbd->dma = dma;
1307         xdp_tx_swbd->dir = DMA_TO_DEVICE;
1308         xdp_tx_swbd->len = len;
1309         xdp_tx_swbd->is_xdp_redirect = true;
1310         xdp_tx_swbd->is_eof = false;
1311         xdp_tx_swbd->xdp_frame = NULL;
1312
1313         n++;
1314         xdp_tx_swbd = &xdp_tx_arr[n];
1315
1316         shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1317
1318         for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1319              f++, frag++) {
1320                 data = skb_frag_address(frag);
1321                 len = skb_frag_size(frag);
1322
1323                 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1324                 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1325                         /* Undo the DMA mapping for all fragments */
1326                         while (--n >= 0)
1327                                 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1328
1329                         netdev_err(tx_ring->ndev, "DMA map error\n");
1330                         return -1;
1331                 }
1332
1333                 xdp_tx_swbd->dma = dma;
1334                 xdp_tx_swbd->dir = DMA_TO_DEVICE;
1335                 xdp_tx_swbd->len = len;
1336                 xdp_tx_swbd->is_xdp_redirect = true;
1337                 xdp_tx_swbd->is_eof = false;
1338                 xdp_tx_swbd->xdp_frame = NULL;
1339
1340                 n++;
1341                 xdp_tx_swbd = &xdp_tx_arr[n];
1342         }
1343
1344         xdp_tx_arr[n - 1].is_eof = true;
1345         xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1346
1347         return n;
1348 }
1349
1350 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1351                    struct xdp_frame **frames, u32 flags)
1352 {
1353         struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1354         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1355         struct enetc_bdr *tx_ring;
1356         int xdp_tx_bd_cnt, i, k;
1357         int xdp_tx_frm_cnt = 0;
1358
1359         enetc_lock_mdio();
1360
1361         tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1362
1363         prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1364
1365         for (k = 0; k < num_frames; k++) {
1366                 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1367                                                                xdp_redirect_arr,
1368                                                                frames[k]);
1369                 if (unlikely(xdp_tx_bd_cnt < 0))
1370                         break;
1371
1372                 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1373                                            xdp_tx_bd_cnt))) {
1374                         for (i = 0; i < xdp_tx_bd_cnt; i++)
1375                                 enetc_unmap_tx_buff(tx_ring,
1376                                                     &xdp_redirect_arr[i]);
1377                         tx_ring->stats.xdp_tx_drops++;
1378                         break;
1379                 }
1380
1381                 xdp_tx_frm_cnt++;
1382         }
1383
1384         if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1385                 enetc_update_tx_ring_tail(tx_ring);
1386
1387         tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1388
1389         enetc_unlock_mdio();
1390
1391         return xdp_tx_frm_cnt;
1392 }
1393
1394 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1395                                      struct xdp_buff *xdp_buff, u16 size)
1396 {
1397         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1398         void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1399         struct skb_shared_info *shinfo;
1400
1401         /* To be used for XDP_TX */
1402         rx_swbd->len = size;
1403
1404         xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1405                          rx_ring->buffer_offset, size, false);
1406
1407         shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1408         shinfo->nr_frags = 0;
1409 }
1410
1411 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1412                                      u16 size, struct xdp_buff *xdp_buff)
1413 {
1414         struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1415         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1416         skb_frag_t *frag = &shinfo->frags[shinfo->nr_frags];
1417
1418         /* To be used for XDP_TX */
1419         rx_swbd->len = size;
1420
1421         skb_frag_off_set(frag, rx_swbd->page_offset);
1422         skb_frag_size_set(frag, size);
1423         __skb_frag_set_page(frag, rx_swbd->page);
1424
1425         shinfo->nr_frags++;
1426 }
1427
1428 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1429                                  union enetc_rx_bd **rxbd, int *i,
1430                                  int *cleaned_cnt, struct xdp_buff *xdp_buff)
1431 {
1432         u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1433
1434         xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1435
1436         enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1437         (*cleaned_cnt)++;
1438         enetc_rxbd_next(rx_ring, rxbd, i);
1439
1440         /* not last BD in frame? */
1441         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1442                 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1443                 size = ENETC_RXB_DMA_SIZE_XDP;
1444
1445                 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1446                         dma_rmb();
1447                         size = le16_to_cpu((*rxbd)->r.buf_len);
1448                 }
1449
1450                 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1451                 (*cleaned_cnt)++;
1452                 enetc_rxbd_next(rx_ring, rxbd, i);
1453         }
1454 }
1455
1456 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1457  * recycled back into the RX ring in enetc_clean_tx_ring.
1458  */
1459 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1460                                         struct enetc_bdr *rx_ring,
1461                                         int rx_ring_first, int rx_ring_last)
1462 {
1463         int n = 0;
1464
1465         for (; rx_ring_first != rx_ring_last;
1466              n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1467                 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1468                 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1469
1470                 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1471                 tx_swbd->dma = rx_swbd->dma;
1472                 tx_swbd->dir = rx_swbd->dir;
1473                 tx_swbd->page = rx_swbd->page;
1474                 tx_swbd->page_offset = rx_swbd->page_offset;
1475                 tx_swbd->len = rx_swbd->len;
1476                 tx_swbd->is_dma_page = true;
1477                 tx_swbd->is_xdp_tx = true;
1478                 tx_swbd->is_eof = false;
1479         }
1480
1481         /* We rely on caller providing an rx_ring_last > rx_ring_first */
1482         xdp_tx_arr[n - 1].is_eof = true;
1483
1484         return n;
1485 }
1486
1487 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1488                            int rx_ring_last)
1489 {
1490         while (rx_ring_first != rx_ring_last) {
1491                 enetc_put_rx_buff(rx_ring,
1492                                   &rx_ring->rx_swbd[rx_ring_first]);
1493                 enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1494         }
1495         rx_ring->stats.xdp_drops++;
1496 }
1497
1498 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1499                                    struct napi_struct *napi, int work_limit,
1500                                    struct bpf_prog *prog)
1501 {
1502         int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1503         struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1504         struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1505         int rx_frm_cnt = 0, rx_byte_cnt = 0;
1506         struct enetc_bdr *tx_ring;
1507         int cleaned_cnt, i;
1508         u32 xdp_act;
1509
1510         cleaned_cnt = enetc_bd_unused(rx_ring);
1511         /* next descriptor to process */
1512         i = rx_ring->next_to_clean;
1513
1514         while (likely(rx_frm_cnt < work_limit)) {
1515                 union enetc_rx_bd *rxbd, *orig_rxbd;
1516                 int orig_i, orig_cleaned_cnt;
1517                 struct xdp_buff xdp_buff;
1518                 struct sk_buff *skb;
1519                 u32 bd_status;
1520                 int err;
1521
1522                 rxbd = enetc_rxbd(rx_ring, i);
1523                 bd_status = le32_to_cpu(rxbd->r.lstatus);
1524                 if (!bd_status)
1525                         break;
1526
1527                 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1528                 dma_rmb(); /* for reading other rxbd fields */
1529
1530                 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1531                                                       &rxbd, &i))
1532                         break;
1533
1534                 orig_rxbd = rxbd;
1535                 orig_cleaned_cnt = cleaned_cnt;
1536                 orig_i = i;
1537
1538                 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1539                                      &cleaned_cnt, &xdp_buff);
1540
1541                 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1542
1543                 switch (xdp_act) {
1544                 default:
1545                         bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1546                         fallthrough;
1547                 case XDP_ABORTED:
1548                         trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1549                         fallthrough;
1550                 case XDP_DROP:
1551                         enetc_xdp_drop(rx_ring, orig_i, i);
1552                         break;
1553                 case XDP_PASS:
1554                         rxbd = orig_rxbd;
1555                         cleaned_cnt = orig_cleaned_cnt;
1556                         i = orig_i;
1557
1558                         skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1559                                               &i, &cleaned_cnt,
1560                                               ENETC_RXB_DMA_SIZE_XDP);
1561                         if (unlikely(!skb))
1562                                 goto out;
1563
1564                         napi_gro_receive(napi, skb);
1565                         break;
1566                 case XDP_TX:
1567                         tx_ring = priv->xdp_tx_ring[rx_ring->index];
1568                         xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1569                                                                      rx_ring,
1570                                                                      orig_i, i);
1571
1572                         if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1573                                 enetc_xdp_drop(rx_ring, orig_i, i);
1574                                 tx_ring->stats.xdp_tx_drops++;
1575                         } else {
1576                                 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
1577                                 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1578                                 xdp_tx_frm_cnt++;
1579                                 /* The XDP_TX enqueue was successful, so we
1580                                  * need to scrub the RX software BDs because
1581                                  * the ownership of the buffers no longer
1582                                  * belongs to the RX ring, and we must prevent
1583                                  * enetc_refill_rx_ring() from reusing
1584                                  * rx_swbd->page.
1585                                  */
1586                                 while (orig_i != i) {
1587                                         rx_ring->rx_swbd[orig_i].page = NULL;
1588                                         enetc_bdr_idx_inc(rx_ring, &orig_i);
1589                                 }
1590                         }
1591                         break;
1592                 case XDP_REDIRECT:
1593                         /* xdp_return_frame does not support S/G in the sense
1594                          * that it leaks the fragments (__xdp_return should not
1595                          * call page_frag_free only for the initial buffer).
1596                          * Until XDP_REDIRECT gains support for S/G let's keep
1597                          * the code structure in place, but dead. We drop the
1598                          * S/G frames ourselves to avoid memory leaks which
1599                          * would otherwise leave the kernel OOM.
1600                          */
1601                         if (unlikely(cleaned_cnt - orig_cleaned_cnt != 1)) {
1602                                 enetc_xdp_drop(rx_ring, orig_i, i);
1603                                 rx_ring->stats.xdp_redirect_sg++;
1604                                 break;
1605                         }
1606
1607                         err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1608                         if (unlikely(err)) {
1609                                 enetc_xdp_drop(rx_ring, orig_i, i);
1610                                 rx_ring->stats.xdp_redirect_failures++;
1611                         } else {
1612                                 while (orig_i != i) {
1613                                         enetc_flip_rx_buff(rx_ring,
1614                                                            &rx_ring->rx_swbd[orig_i]);
1615                                         enetc_bdr_idx_inc(rx_ring, &orig_i);
1616                                 }
1617                                 xdp_redirect_frm_cnt++;
1618                                 rx_ring->stats.xdp_redirect++;
1619                         }
1620                 }
1621
1622                 rx_frm_cnt++;
1623         }
1624
1625 out:
1626         rx_ring->next_to_clean = i;
1627
1628         rx_ring->stats.packets += rx_frm_cnt;
1629         rx_ring->stats.bytes += rx_byte_cnt;
1630
1631         if (xdp_redirect_frm_cnt)
1632                 xdp_do_flush_map();
1633
1634         if (xdp_tx_frm_cnt)
1635                 enetc_update_tx_ring_tail(tx_ring);
1636
1637         if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
1638                 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
1639                                      rx_ring->xdp.xdp_tx_in_flight);
1640
1641         return rx_frm_cnt;
1642 }
1643
1644 static int enetc_poll(struct napi_struct *napi, int budget)
1645 {
1646         struct enetc_int_vector
1647                 *v = container_of(napi, struct enetc_int_vector, napi);
1648         struct enetc_bdr *rx_ring = &v->rx_ring;
1649         struct bpf_prog *prog;
1650         bool complete = true;
1651         int work_done;
1652         int i;
1653
1654         enetc_lock_mdio();
1655
1656         for (i = 0; i < v->count_tx_rings; i++)
1657                 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
1658                         complete = false;
1659
1660         prog = rx_ring->xdp.prog;
1661         if (prog)
1662                 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1663         else
1664                 work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
1665         if (work_done == budget)
1666                 complete = false;
1667         if (work_done)
1668                 v->rx_napi_work = true;
1669
1670         if (!complete) {
1671                 enetc_unlock_mdio();
1672                 return budget;
1673         }
1674
1675         napi_complete_done(napi, work_done);
1676
1677         if (likely(v->rx_dim_en))
1678                 enetc_rx_net_dim(v);
1679
1680         v->rx_napi_work = false;
1681
1682         /* enable interrupts */
1683         enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
1684
1685         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1686                 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
1687                                  ENETC_TBIER_TXTIE);
1688
1689         enetc_unlock_mdio();
1690
1691         return work_done;
1692 }
1693
1694 /* Probing and Init */
1695 #define ENETC_MAX_RFS_SIZE 64
1696 void enetc_get_si_caps(struct enetc_si *si)
1697 {
1698         struct enetc_hw *hw = &si->hw;
1699         u32 val;
1700
1701         /* find out how many of various resources we have to work with */
1702         val = enetc_rd(hw, ENETC_SICAPR0);
1703         si->num_rx_rings = (val >> 16) & 0xff;
1704         si->num_tx_rings = val & 0xff;
1705
1706         val = enetc_rd(hw, ENETC_SIRFSCAPR);
1707         si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1708         si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1709
1710         si->num_rss = 0;
1711         val = enetc_rd(hw, ENETC_SIPCAPR0);
1712         if (val & ENETC_SIPCAPR0_RSS) {
1713                 u32 rss;
1714
1715                 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
1716                 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1717         }
1718
1719         if (val & ENETC_SIPCAPR0_QBV)
1720                 si->hw_features |= ENETC_SI_F_QBV;
1721
1722         if (val & ENETC_SIPCAPR0_PSFP)
1723                 si->hw_features |= ENETC_SI_F_PSFP;
1724 }
1725
1726 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
1727 {
1728         r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
1729                                         &r->bd_dma_base, GFP_KERNEL);
1730         if (!r->bd_base)
1731                 return -ENOMEM;
1732
1733         /* h/w requires 128B alignment */
1734         if (!IS_ALIGNED(r->bd_dma_base, 128)) {
1735                 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
1736                                   r->bd_dma_base);
1737                 return -EINVAL;
1738         }
1739
1740         return 0;
1741 }
1742
1743 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
1744 {
1745         int err;
1746
1747         txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
1748         if (!txr->tx_swbd)
1749                 return -ENOMEM;
1750
1751         err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
1752         if (err)
1753                 goto err_alloc_bdr;
1754
1755         txr->tso_headers = dma_alloc_coherent(txr->dev,
1756                                               txr->bd_count * TSO_HEADER_SIZE,
1757                                               &txr->tso_headers_dma,
1758                                               GFP_KERNEL);
1759         if (!txr->tso_headers) {
1760                 err = -ENOMEM;
1761                 goto err_alloc_tso;
1762         }
1763
1764         txr->next_to_clean = 0;
1765         txr->next_to_use = 0;
1766
1767         return 0;
1768
1769 err_alloc_tso:
1770         dma_free_coherent(txr->dev, txr->bd_count * sizeof(union enetc_tx_bd),
1771                           txr->bd_base, txr->bd_dma_base);
1772         txr->bd_base = NULL;
1773 err_alloc_bdr:
1774         vfree(txr->tx_swbd);
1775         txr->tx_swbd = NULL;
1776
1777         return err;
1778 }
1779
1780 static void enetc_free_txbdr(struct enetc_bdr *txr)
1781 {
1782         int size, i;
1783
1784         for (i = 0; i < txr->bd_count; i++)
1785                 enetc_free_tx_frame(txr, &txr->tx_swbd[i]);
1786
1787         size = txr->bd_count * sizeof(union enetc_tx_bd);
1788
1789         dma_free_coherent(txr->dev, txr->bd_count * TSO_HEADER_SIZE,
1790                           txr->tso_headers, txr->tso_headers_dma);
1791         txr->tso_headers = NULL;
1792
1793         dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
1794         txr->bd_base = NULL;
1795
1796         vfree(txr->tx_swbd);
1797         txr->tx_swbd = NULL;
1798 }
1799
1800 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1801 {
1802         int i, err;
1803
1804         for (i = 0; i < priv->num_tx_rings; i++) {
1805                 err = enetc_alloc_txbdr(priv->tx_ring[i]);
1806
1807                 if (err)
1808                         goto fail;
1809         }
1810
1811         return 0;
1812
1813 fail:
1814         while (i-- > 0)
1815                 enetc_free_txbdr(priv->tx_ring[i]);
1816
1817         return err;
1818 }
1819
1820 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
1821 {
1822         int i;
1823
1824         for (i = 0; i < priv->num_tx_rings; i++)
1825                 enetc_free_txbdr(priv->tx_ring[i]);
1826 }
1827
1828 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
1829 {
1830         size_t size = sizeof(union enetc_rx_bd);
1831         int err;
1832
1833         rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
1834         if (!rxr->rx_swbd)
1835                 return -ENOMEM;
1836
1837         if (extended)
1838                 size *= 2;
1839
1840         err = enetc_dma_alloc_bdr(rxr, size);
1841         if (err) {
1842                 vfree(rxr->rx_swbd);
1843                 return err;
1844         }
1845
1846         rxr->next_to_clean = 0;
1847         rxr->next_to_use = 0;
1848         rxr->next_to_alloc = 0;
1849         rxr->ext_en = extended;
1850
1851         return 0;
1852 }
1853
1854 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
1855 {
1856         int size;
1857
1858         size = rxr->bd_count * sizeof(union enetc_rx_bd);
1859
1860         dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
1861         rxr->bd_base = NULL;
1862
1863         vfree(rxr->rx_swbd);
1864         rxr->rx_swbd = NULL;
1865 }
1866
1867 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
1868 {
1869         bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
1870         int i, err;
1871
1872         for (i = 0; i < priv->num_rx_rings; i++) {
1873                 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
1874
1875                 if (err)
1876                         goto fail;
1877         }
1878
1879         return 0;
1880
1881 fail:
1882         while (i-- > 0)
1883                 enetc_free_rxbdr(priv->rx_ring[i]);
1884
1885         return err;
1886 }
1887
1888 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
1889 {
1890         int i;
1891
1892         for (i = 0; i < priv->num_rx_rings; i++)
1893                 enetc_free_rxbdr(priv->rx_ring[i]);
1894 }
1895
1896 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
1897 {
1898         int i;
1899
1900         if (!tx_ring->tx_swbd)
1901                 return;
1902
1903         for (i = 0; i < tx_ring->bd_count; i++) {
1904                 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
1905
1906                 enetc_free_tx_frame(tx_ring, tx_swbd);
1907         }
1908
1909         tx_ring->next_to_clean = 0;
1910         tx_ring->next_to_use = 0;
1911 }
1912
1913 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
1914 {
1915         int i;
1916
1917         if (!rx_ring->rx_swbd)
1918                 return;
1919
1920         for (i = 0; i < rx_ring->bd_count; i++) {
1921                 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1922
1923                 if (!rx_swbd->page)
1924                         continue;
1925
1926                 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1927                                rx_swbd->dir);
1928                 __free_page(rx_swbd->page);
1929                 rx_swbd->page = NULL;
1930         }
1931
1932         rx_ring->next_to_clean = 0;
1933         rx_ring->next_to_use = 0;
1934         rx_ring->next_to_alloc = 0;
1935 }
1936
1937 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
1938 {
1939         int i;
1940
1941         for (i = 0; i < priv->num_rx_rings; i++)
1942                 enetc_free_rx_ring(priv->rx_ring[i]);
1943
1944         for (i = 0; i < priv->num_tx_rings; i++)
1945                 enetc_free_tx_ring(priv->tx_ring[i]);
1946 }
1947
1948 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1949 {
1950         int *rss_table;
1951         int i;
1952
1953         rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1954         if (!rss_table)
1955                 return -ENOMEM;
1956
1957         /* Set up RSS table defaults */
1958         for (i = 0; i < si->num_rss; i++)
1959                 rss_table[i] = i % num_groups;
1960
1961         enetc_set_rss_table(si, rss_table, si->num_rss);
1962
1963         kfree(rss_table);
1964
1965         return 0;
1966 }
1967
1968 int enetc_configure_si(struct enetc_ndev_priv *priv)
1969 {
1970         struct enetc_si *si = priv->si;
1971         struct enetc_hw *hw = &si->hw;
1972         int err;
1973
1974         /* set SI cache attributes */
1975         enetc_wr(hw, ENETC_SICAR0,
1976                  ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1977         enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1978         /* enable SI */
1979         enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1980
1981         if (si->num_rss) {
1982                 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1983                 if (err)
1984                         return err;
1985         }
1986
1987         return 0;
1988 }
1989
1990 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1991 {
1992         struct enetc_si *si = priv->si;
1993         int cpus = num_online_cpus();
1994
1995         priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
1996         priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
1997
1998         /* Enable all available TX rings in order to configure as many
1999          * priorities as possible, when needed.
2000          * TODO: Make # of TX rings run-time configurable
2001          */
2002         priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2003         priv->num_tx_rings = si->num_tx_rings;
2004         priv->bdr_int_num = cpus;
2005         priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2006         priv->tx_ictt = ENETC_TXIC_TIMETHR;
2007 }
2008
2009 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2010 {
2011         struct enetc_si *si = priv->si;
2012
2013         priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2014                                   GFP_KERNEL);
2015         if (!priv->cls_rules)
2016                 return -ENOMEM;
2017
2018         return 0;
2019 }
2020
2021 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2022 {
2023         kfree(priv->cls_rules);
2024 }
2025
2026 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2027 {
2028         int idx = tx_ring->index;
2029         u32 tbmr;
2030
2031         enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2032                        lower_32_bits(tx_ring->bd_dma_base));
2033
2034         enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2035                        upper_32_bits(tx_ring->bd_dma_base));
2036
2037         WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2038         enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2039                        ENETC_RTBLENR_LEN(tx_ring->bd_count));
2040
2041         /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2042         tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2043         tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2044
2045         /* enable Tx ints by setting pkt thr to 1 */
2046         enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2047
2048         tbmr = ENETC_TBMR_EN | ENETC_TBMR_SET_PRIO(tx_ring->prio);
2049         if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2050                 tbmr |= ENETC_TBMR_VIH;
2051
2052         /* enable ring */
2053         enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2054
2055         tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2056         tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2057         tx_ring->idr = hw->reg + ENETC_SITXIDR;
2058 }
2059
2060 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2061 {
2062         int idx = rx_ring->index;
2063         u32 rbmr;
2064
2065         enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2066                        lower_32_bits(rx_ring->bd_dma_base));
2067
2068         enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2069                        upper_32_bits(rx_ring->bd_dma_base));
2070
2071         WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2072         enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2073                        ENETC_RTBLENR_LEN(rx_ring->bd_count));
2074
2075         if (rx_ring->xdp.prog)
2076                 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2077         else
2078                 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2079
2080         /* Also prepare the consumer index in case page allocation never
2081          * succeeds. In that case, hardware will never advance producer index
2082          * to match consumer index, and will drop all frames.
2083          */
2084         enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2085         enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2086
2087         /* enable Rx ints by setting pkt thr to 1 */
2088         enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2089
2090         rbmr = ENETC_RBMR_EN;
2091
2092         if (rx_ring->ext_en)
2093                 rbmr |= ENETC_RBMR_BDS;
2094
2095         if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2096                 rbmr |= ENETC_RBMR_VTE;
2097
2098         rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2099         rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2100
2101         enetc_lock_mdio();
2102         enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2103         enetc_unlock_mdio();
2104
2105         /* enable ring */
2106         enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2107 }
2108
2109 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
2110 {
2111         struct enetc_hw *hw = &priv->si->hw;
2112         int i;
2113
2114         for (i = 0; i < priv->num_tx_rings; i++)
2115                 enetc_setup_txbdr(hw, priv->tx_ring[i]);
2116
2117         for (i = 0; i < priv->num_rx_rings; i++)
2118                 enetc_setup_rxbdr(hw, priv->rx_ring[i]);
2119 }
2120
2121 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2122 {
2123         int idx = rx_ring->index;
2124
2125         /* disable EN bit on ring */
2126         enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2127 }
2128
2129 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2130 {
2131         int delay = 8, timeout = 100;
2132         int idx = tx_ring->index;
2133
2134         /* disable EN bit on ring */
2135         enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2136
2137         /* wait for busy to clear */
2138         while (delay < timeout &&
2139                enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2140                 msleep(delay);
2141                 delay *= 2;
2142         }
2143
2144         if (delay >= timeout)
2145                 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2146                             idx);
2147 }
2148
2149 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
2150 {
2151         struct enetc_hw *hw = &priv->si->hw;
2152         int i;
2153
2154         for (i = 0; i < priv->num_tx_rings; i++)
2155                 enetc_clear_txbdr(hw, priv->tx_ring[i]);
2156
2157         for (i = 0; i < priv->num_rx_rings; i++)
2158                 enetc_clear_rxbdr(hw, priv->rx_ring[i]);
2159
2160         udelay(1);
2161 }
2162
2163 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2164 {
2165         struct pci_dev *pdev = priv->si->pdev;
2166         struct enetc_hw *hw = &priv->si->hw;
2167         int i, j, err;
2168
2169         for (i = 0; i < priv->bdr_int_num; i++) {
2170                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2171                 struct enetc_int_vector *v = priv->int_vector[i];
2172                 int entry = ENETC_BDR_INT_BASE_IDX + i;
2173
2174                 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2175                          priv->ndev->name, i);
2176                 err = request_irq(irq, enetc_msix, 0, v->name, v);
2177                 if (err) {
2178                         dev_err(priv->dev, "request_irq() failed!\n");
2179                         goto irq_err;
2180                 }
2181                 disable_irq(irq);
2182
2183                 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2184                 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2185                 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2186
2187                 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2188
2189                 for (j = 0; j < v->count_tx_rings; j++) {
2190                         int idx = v->tx_ring[j].index;
2191
2192                         enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2193                 }
2194                 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2195         }
2196
2197         return 0;
2198
2199 irq_err:
2200         while (i--) {
2201                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2202
2203                 irq_set_affinity_hint(irq, NULL);
2204                 free_irq(irq, priv->int_vector[i]);
2205         }
2206
2207         return err;
2208 }
2209
2210 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2211 {
2212         struct pci_dev *pdev = priv->si->pdev;
2213         int i;
2214
2215         for (i = 0; i < priv->bdr_int_num; i++) {
2216                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2217
2218                 irq_set_affinity_hint(irq, NULL);
2219                 free_irq(irq, priv->int_vector[i]);
2220         }
2221 }
2222
2223 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2224 {
2225         struct enetc_hw *hw = &priv->si->hw;
2226         u32 icpt, ictt;
2227         int i;
2228
2229         /* enable Tx & Rx event indication */
2230         if (priv->ic_mode &
2231             (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2232                 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2233                 /* init to non-0 minimum, will be adjusted later */
2234                 ictt = 0x1;
2235         } else {
2236                 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2237                 ictt = 0;
2238         }
2239
2240         for (i = 0; i < priv->num_rx_rings; i++) {
2241                 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2242                 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2243                 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2244         }
2245
2246         if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2247                 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2248         else
2249                 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2250
2251         for (i = 0; i < priv->num_tx_rings; i++) {
2252                 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2253                 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2254                 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2255         }
2256 }
2257
2258 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2259 {
2260         struct enetc_hw *hw = &priv->si->hw;
2261         int i;
2262
2263         for (i = 0; i < priv->num_tx_rings; i++)
2264                 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2265
2266         for (i = 0; i < priv->num_rx_rings; i++)
2267                 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2268 }
2269
2270 static int enetc_phylink_connect(struct net_device *ndev)
2271 {
2272         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2273         struct ethtool_eee edata;
2274         int err;
2275
2276         if (!priv->phylink)
2277                 return 0; /* phy-less mode */
2278
2279         err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2280         if (err) {
2281                 dev_err(&ndev->dev, "could not attach to PHY\n");
2282                 return err;
2283         }
2284
2285         /* disable EEE autoneg, until ENETC driver supports it */
2286         memset(&edata, 0, sizeof(struct ethtool_eee));
2287         phylink_ethtool_set_eee(priv->phylink, &edata);
2288
2289         return 0;
2290 }
2291
2292 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2293 {
2294         struct enetc_ndev_priv *priv;
2295         struct sk_buff *skb;
2296
2297         priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2298
2299         netif_tx_lock_bh(priv->ndev);
2300
2301         clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2302         skb = skb_dequeue(&priv->tx_skbs);
2303         if (skb)
2304                 enetc_start_xmit(skb, priv->ndev);
2305
2306         netif_tx_unlock_bh(priv->ndev);
2307 }
2308
2309 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2310 {
2311         INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2312         skb_queue_head_init(&priv->tx_skbs);
2313 }
2314
2315 void enetc_start(struct net_device *ndev)
2316 {
2317         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2318         int i;
2319
2320         enetc_setup_interrupts(priv);
2321
2322         for (i = 0; i < priv->bdr_int_num; i++) {
2323                 int irq = pci_irq_vector(priv->si->pdev,
2324                                          ENETC_BDR_INT_BASE_IDX + i);
2325
2326                 napi_enable(&priv->int_vector[i]->napi);
2327                 enable_irq(irq);
2328         }
2329
2330         if (priv->phylink)
2331                 phylink_start(priv->phylink);
2332         else
2333                 netif_carrier_on(ndev);
2334
2335         netif_tx_start_all_queues(ndev);
2336 }
2337
2338 int enetc_open(struct net_device *ndev)
2339 {
2340         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2341         int num_stack_tx_queues;
2342         int err;
2343
2344         err = enetc_setup_irqs(priv);
2345         if (err)
2346                 return err;
2347
2348         err = enetc_phylink_connect(ndev);
2349         if (err)
2350                 goto err_phy_connect;
2351
2352         err = enetc_alloc_tx_resources(priv);
2353         if (err)
2354                 goto err_alloc_tx;
2355
2356         err = enetc_alloc_rx_resources(priv);
2357         if (err)
2358                 goto err_alloc_rx;
2359
2360         num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2361
2362         err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2363         if (err)
2364                 goto err_set_queues;
2365
2366         err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
2367         if (err)
2368                 goto err_set_queues;
2369
2370         enetc_tx_onestep_tstamp_init(priv);
2371         enetc_setup_bdrs(priv);
2372         enetc_start(ndev);
2373
2374         return 0;
2375
2376 err_set_queues:
2377         enetc_free_rx_resources(priv);
2378 err_alloc_rx:
2379         enetc_free_tx_resources(priv);
2380 err_alloc_tx:
2381         if (priv->phylink)
2382                 phylink_disconnect_phy(priv->phylink);
2383 err_phy_connect:
2384         enetc_free_irqs(priv);
2385
2386         return err;
2387 }
2388
2389 void enetc_stop(struct net_device *ndev)
2390 {
2391         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2392         int i;
2393
2394         netif_tx_stop_all_queues(ndev);
2395
2396         for (i = 0; i < priv->bdr_int_num; i++) {
2397                 int irq = pci_irq_vector(priv->si->pdev,
2398                                          ENETC_BDR_INT_BASE_IDX + i);
2399
2400                 disable_irq(irq);
2401                 napi_synchronize(&priv->int_vector[i]->napi);
2402                 napi_disable(&priv->int_vector[i]->napi);
2403         }
2404
2405         if (priv->phylink)
2406                 phylink_stop(priv->phylink);
2407         else
2408                 netif_carrier_off(ndev);
2409
2410         enetc_clear_interrupts(priv);
2411 }
2412
2413 int enetc_close(struct net_device *ndev)
2414 {
2415         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2416
2417         enetc_stop(ndev);
2418         enetc_clear_bdrs(priv);
2419
2420         if (priv->phylink)
2421                 phylink_disconnect_phy(priv->phylink);
2422         enetc_free_rxtx_rings(priv);
2423         enetc_free_rx_resources(priv);
2424         enetc_free_tx_resources(priv);
2425         enetc_free_irqs(priv);
2426
2427         return 0;
2428 }
2429
2430 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
2431 {
2432         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2433         struct tc_mqprio_qopt *mqprio = type_data;
2434         struct enetc_hw *hw = &priv->si->hw;
2435         struct enetc_bdr *tx_ring;
2436         int num_stack_tx_queues;
2437         u8 num_tc;
2438         int i;
2439
2440         num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2441         mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
2442         num_tc = mqprio->num_tc;
2443
2444         if (!num_tc) {
2445                 netdev_reset_tc(ndev);
2446                 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2447
2448                 /* Reset all ring priorities to 0 */
2449                 for (i = 0; i < priv->num_tx_rings; i++) {
2450                         tx_ring = priv->tx_ring[i];
2451                         tx_ring->prio = 0;
2452                         enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2453                 }
2454
2455                 return 0;
2456         }
2457
2458         /* Check if we have enough BD rings available to accommodate all TCs */
2459         if (num_tc > num_stack_tx_queues) {
2460                 netdev_err(ndev, "Max %d traffic classes supported\n",
2461                            priv->num_tx_rings);
2462                 return -EINVAL;
2463         }
2464
2465         /* For the moment, we use only one BD ring per TC.
2466          *
2467          * Configure num_tc BD rings with increasing priorities.
2468          */
2469         for (i = 0; i < num_tc; i++) {
2470                 tx_ring = priv->tx_ring[i];
2471                 tx_ring->prio = i;
2472                 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2473         }
2474
2475         /* Reset the number of netdev queues based on the TC count */
2476         netif_set_real_num_tx_queues(ndev, num_tc);
2477
2478         netdev_set_num_tc(ndev, num_tc);
2479
2480         /* Each TC is associated with one netdev queue */
2481         for (i = 0; i < num_tc; i++)
2482                 netdev_set_tc_queue(ndev, i, 1, i);
2483
2484         return 0;
2485 }
2486
2487 static int enetc_setup_xdp_prog(struct net_device *dev, struct bpf_prog *prog,
2488                                 struct netlink_ext_ack *extack)
2489 {
2490         struct enetc_ndev_priv *priv = netdev_priv(dev);
2491         struct bpf_prog *old_prog;
2492         bool is_up;
2493         int i;
2494
2495         /* The buffer layout is changing, so we need to drain the old
2496          * RX buffers and seed new ones.
2497          */
2498         is_up = netif_running(dev);
2499         if (is_up)
2500                 dev_close(dev);
2501
2502         old_prog = xchg(&priv->xdp_prog, prog);
2503         if (old_prog)
2504                 bpf_prog_put(old_prog);
2505
2506         for (i = 0; i < priv->num_rx_rings; i++) {
2507                 struct enetc_bdr *rx_ring = priv->rx_ring[i];
2508
2509                 rx_ring->xdp.prog = prog;
2510
2511                 if (prog)
2512                         rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2513                 else
2514                         rx_ring->buffer_offset = ENETC_RXB_PAD;
2515         }
2516
2517         if (is_up)
2518                 return dev_open(dev, extack);
2519
2520         return 0;
2521 }
2522
2523 int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp)
2524 {
2525         switch (xdp->command) {
2526         case XDP_SETUP_PROG:
2527                 return enetc_setup_xdp_prog(dev, xdp->prog, xdp->extack);
2528         default:
2529                 return -EINVAL;
2530         }
2531
2532         return 0;
2533 }
2534
2535 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2536 {
2537         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2538         struct net_device_stats *stats = &ndev->stats;
2539         unsigned long packets = 0, bytes = 0;
2540         unsigned long tx_dropped = 0;
2541         int i;
2542
2543         for (i = 0; i < priv->num_rx_rings; i++) {
2544                 packets += priv->rx_ring[i]->stats.packets;
2545                 bytes   += priv->rx_ring[i]->stats.bytes;
2546         }
2547
2548         stats->rx_packets = packets;
2549         stats->rx_bytes = bytes;
2550         bytes = 0;
2551         packets = 0;
2552
2553         for (i = 0; i < priv->num_tx_rings; i++) {
2554                 packets += priv->tx_ring[i]->stats.packets;
2555                 bytes   += priv->tx_ring[i]->stats.bytes;
2556                 tx_dropped += priv->tx_ring[i]->stats.win_drop;
2557         }
2558
2559         stats->tx_packets = packets;
2560         stats->tx_bytes = bytes;
2561         stats->tx_dropped = tx_dropped;
2562
2563         return stats;
2564 }
2565
2566 static int enetc_set_rss(struct net_device *ndev, int en)
2567 {
2568         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2569         struct enetc_hw *hw = &priv->si->hw;
2570         u32 reg;
2571
2572         enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2573
2574         reg = enetc_rd(hw, ENETC_SIMR);
2575         reg &= ~ENETC_SIMR_RSSE;
2576         reg |= (en) ? ENETC_SIMR_RSSE : 0;
2577         enetc_wr(hw, ENETC_SIMR, reg);
2578
2579         return 0;
2580 }
2581
2582 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
2583 {
2584         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2585         struct enetc_hw *hw = &priv->si->hw;
2586         int i;
2587
2588         for (i = 0; i < priv->num_rx_rings; i++)
2589                 enetc_bdr_enable_rxvlan(hw, i, en);
2590 }
2591
2592 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
2593 {
2594         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2595         struct enetc_hw *hw = &priv->si->hw;
2596         int i;
2597
2598         for (i = 0; i < priv->num_tx_rings; i++)
2599                 enetc_bdr_enable_txvlan(hw, i, en);
2600 }
2601
2602 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
2603 {
2604         netdev_features_t changed = ndev->features ^ features;
2605
2606         if (changed & NETIF_F_RXHASH)
2607                 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2608
2609         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2610                 enetc_enable_rxvlan(ndev,
2611                                     !!(features & NETIF_F_HW_VLAN_CTAG_RX));
2612
2613         if (changed & NETIF_F_HW_VLAN_CTAG_TX)
2614                 enetc_enable_txvlan(ndev,
2615                                     !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2616 }
2617
2618 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2619 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2620 {
2621         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2622         struct hwtstamp_config config;
2623         int ao;
2624
2625         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2626                 return -EFAULT;
2627
2628         switch (config.tx_type) {
2629         case HWTSTAMP_TX_OFF:
2630                 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2631                 break;
2632         case HWTSTAMP_TX_ON:
2633                 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2634                 priv->active_offloads |= ENETC_F_TX_TSTAMP;
2635                 break;
2636         case HWTSTAMP_TX_ONESTEP_SYNC:
2637                 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2638                 priv->active_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
2639                 break;
2640         default:
2641                 return -ERANGE;
2642         }
2643
2644         ao = priv->active_offloads;
2645         switch (config.rx_filter) {
2646         case HWTSTAMP_FILTER_NONE:
2647                 priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
2648                 break;
2649         default:
2650                 priv->active_offloads |= ENETC_F_RX_TSTAMP;
2651                 config.rx_filter = HWTSTAMP_FILTER_ALL;
2652         }
2653
2654         if (netif_running(ndev) && ao != priv->active_offloads) {
2655                 enetc_close(ndev);
2656                 enetc_open(ndev);
2657         }
2658
2659         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2660                -EFAULT : 0;
2661 }
2662
2663 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2664 {
2665         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2666         struct hwtstamp_config config;
2667
2668         config.flags = 0;
2669
2670         if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
2671                 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
2672         else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2673                 config.tx_type = HWTSTAMP_TX_ON;
2674         else
2675                 config.tx_type = HWTSTAMP_TX_OFF;
2676
2677         config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2678                             HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2679
2680         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2681                -EFAULT : 0;
2682 }
2683 #endif
2684
2685 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2686 {
2687         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2688 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2689         if (cmd == SIOCSHWTSTAMP)
2690                 return enetc_hwtstamp_set(ndev, rq);
2691         if (cmd == SIOCGHWTSTAMP)
2692                 return enetc_hwtstamp_get(ndev, rq);
2693 #endif
2694
2695         if (!priv->phylink)
2696                 return -EOPNOTSUPP;
2697
2698         return phylink_mii_ioctl(priv->phylink, rq, cmd);
2699 }
2700
2701 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
2702 {
2703         struct pci_dev *pdev = priv->si->pdev;
2704         int first_xdp_tx_ring;
2705         int i, n, err, nvec;
2706         int v_tx_rings;
2707
2708         nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
2709         /* allocate MSIX for both messaging and Rx/Tx interrupts */
2710         n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
2711
2712         if (n < 0)
2713                 return n;
2714
2715         if (n != nvec)
2716                 return -EPERM;
2717
2718         /* # of tx rings per int vector */
2719         v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
2720
2721         for (i = 0; i < priv->bdr_int_num; i++) {
2722                 struct enetc_int_vector *v;
2723                 struct enetc_bdr *bdr;
2724                 int j;
2725
2726                 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
2727                 if (!v) {
2728                         err = -ENOMEM;
2729                         goto fail;
2730                 }
2731
2732                 priv->int_vector[i] = v;
2733
2734                 bdr = &v->rx_ring;
2735                 bdr->index = i;
2736                 bdr->ndev = priv->ndev;
2737                 bdr->dev = priv->dev;
2738                 bdr->bd_count = priv->rx_bd_count;
2739                 bdr->buffer_offset = ENETC_RXB_PAD;
2740                 priv->rx_ring[i] = bdr;
2741
2742                 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
2743                 if (err) {
2744                         kfree(v);
2745                         goto fail;
2746                 }
2747
2748                 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
2749                                                  MEM_TYPE_PAGE_SHARED, NULL);
2750                 if (err) {
2751                         xdp_rxq_info_unreg(&bdr->xdp.rxq);
2752                         kfree(v);
2753                         goto fail;
2754                 }
2755
2756                 /* init defaults for adaptive IC */
2757                 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
2758                         v->rx_ictt = 0x1;
2759                         v->rx_dim_en = true;
2760                 }
2761                 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
2762                 netif_napi_add(priv->ndev, &v->napi, enetc_poll);
2763                 v->count_tx_rings = v_tx_rings;
2764
2765                 for (j = 0; j < v_tx_rings; j++) {
2766                         int idx;
2767
2768                         /* default tx ring mapping policy */
2769                         idx = priv->bdr_int_num * j + i;
2770                         __set_bit(idx, &v->tx_rings_map);
2771                         bdr = &v->tx_ring[j];
2772                         bdr->index = idx;
2773                         bdr->ndev = priv->ndev;
2774                         bdr->dev = priv->dev;
2775                         bdr->bd_count = priv->tx_bd_count;
2776                         priv->tx_ring[idx] = bdr;
2777                 }
2778         }
2779
2780         first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
2781         priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
2782
2783         return 0;
2784
2785 fail:
2786         while (i--) {
2787                 struct enetc_int_vector *v = priv->int_vector[i];
2788                 struct enetc_bdr *rx_ring = &v->rx_ring;
2789
2790                 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2791                 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2792                 netif_napi_del(&v->napi);
2793                 cancel_work_sync(&v->rx_dim.work);
2794                 kfree(v);
2795         }
2796
2797         pci_free_irq_vectors(pdev);
2798
2799         return err;
2800 }
2801
2802 void enetc_free_msix(struct enetc_ndev_priv *priv)
2803 {
2804         int i;
2805
2806         for (i = 0; i < priv->bdr_int_num; i++) {
2807                 struct enetc_int_vector *v = priv->int_vector[i];
2808                 struct enetc_bdr *rx_ring = &v->rx_ring;
2809
2810                 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2811                 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2812                 netif_napi_del(&v->napi);
2813                 cancel_work_sync(&v->rx_dim.work);
2814         }
2815
2816         for (i = 0; i < priv->num_rx_rings; i++)
2817                 priv->rx_ring[i] = NULL;
2818
2819         for (i = 0; i < priv->num_tx_rings; i++)
2820                 priv->tx_ring[i] = NULL;
2821
2822         for (i = 0; i < priv->bdr_int_num; i++) {
2823                 kfree(priv->int_vector[i]);
2824                 priv->int_vector[i] = NULL;
2825         }
2826
2827         /* disable all MSIX for this device */
2828         pci_free_irq_vectors(priv->si->pdev);
2829 }
2830
2831 static void enetc_kfree_si(struct enetc_si *si)
2832 {
2833         char *p = (char *)si - si->pad;
2834
2835         kfree(p);
2836 }
2837
2838 static void enetc_detect_errata(struct enetc_si *si)
2839 {
2840         if (si->pdev->revision == ENETC_REV1)
2841                 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
2842 }
2843
2844 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
2845 {
2846         struct enetc_si *si, *p;
2847         struct enetc_hw *hw;
2848         size_t alloc_size;
2849         int err, len;
2850
2851         pcie_flr(pdev);
2852         err = pci_enable_device_mem(pdev);
2853         if (err)
2854                 return dev_err_probe(&pdev->dev, err, "device enable failed\n");
2855
2856         /* set up for high or low dma */
2857         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2858         if (err) {
2859                 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
2860                 goto err_dma;
2861         }
2862
2863         err = pci_request_mem_regions(pdev, name);
2864         if (err) {
2865                 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
2866                 goto err_pci_mem_reg;
2867         }
2868
2869         pci_set_master(pdev);
2870
2871         alloc_size = sizeof(struct enetc_si);
2872         if (sizeof_priv) {
2873                 /* align priv to 32B */
2874                 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
2875                 alloc_size += sizeof_priv;
2876         }
2877         /* force 32B alignment for enetc_si */
2878         alloc_size += ENETC_SI_ALIGN - 1;
2879
2880         p = kzalloc(alloc_size, GFP_KERNEL);
2881         if (!p) {
2882                 err = -ENOMEM;
2883                 goto err_alloc_si;
2884         }
2885
2886         si = PTR_ALIGN(p, ENETC_SI_ALIGN);
2887         si->pad = (char *)si - (char *)p;
2888
2889         pci_set_drvdata(pdev, si);
2890         si->pdev = pdev;
2891         hw = &si->hw;
2892
2893         len = pci_resource_len(pdev, ENETC_BAR_REGS);
2894         hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
2895         if (!hw->reg) {
2896                 err = -ENXIO;
2897                 dev_err(&pdev->dev, "ioremap() failed\n");
2898                 goto err_ioremap;
2899         }
2900         if (len > ENETC_PORT_BASE)
2901                 hw->port = hw->reg + ENETC_PORT_BASE;
2902         if (len > ENETC_GLOBAL_BASE)
2903                 hw->global = hw->reg + ENETC_GLOBAL_BASE;
2904
2905         enetc_detect_errata(si);
2906
2907         return 0;
2908
2909 err_ioremap:
2910         enetc_kfree_si(si);
2911 err_alloc_si:
2912         pci_release_mem_regions(pdev);
2913 err_pci_mem_reg:
2914 err_dma:
2915         pci_disable_device(pdev);
2916
2917         return err;
2918 }
2919
2920 void enetc_pci_remove(struct pci_dev *pdev)
2921 {
2922         struct enetc_si *si = pci_get_drvdata(pdev);
2923         struct enetc_hw *hw = &si->hw;
2924
2925         iounmap(hw->reg);
2926         enetc_kfree_si(si);
2927         pci_release_mem_regions(pdev);
2928         pci_disable_device(pdev);
2929 }