drivers/net/ethernet/freescale/enetc/enetc.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /* Copyright 2017-2019 NXP */
   3
   4 #include "enetc.h"
   5 #include <linux/bpf_trace.h>
   6 #include <linux/tcp.h>
   7 #include <linux/udp.h>
   8 #include <linux/vmalloc.h>
   9 #include <linux/ptp_classify.h>
  10 #include <net/pkt_sched.h>
  11
  12 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
  13 {
  14         int num_tx_rings = priv->num_tx_rings;
  15         int i;
  16
  17         for (i = 0; i < priv->num_rx_rings; i++)
  18                 if (priv->rx_ring[i]->xdp.prog)
  19                         return num_tx_rings - num_possible_cpus();
  20
  21         return num_tx_rings;
  22 }
  23
  24 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
  25                                                         struct enetc_bdr *tx_ring)
  26 {
  27         int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
  28
  29         return priv->rx_ring[index];
  30 }
  31
  32 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
  33 {
  34         if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
  35                 return NULL;
  36
  37         return tx_swbd->skb;
  38 }
  39
  40 static struct xdp_frame *
  41 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
  42 {
  43         if (tx_swbd->is_xdp_redirect)
  44                 return tx_swbd->xdp_frame;
  45
  46         return NULL;
  47 }
  48
  49 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
  50                                 struct enetc_tx_swbd *tx_swbd)
  51 {
  52         /* For XDP_TX, pages come from RX, whereas for the other contexts where
  53          * we have is_dma_page_set, those come from skb_frag_dma_map. We need
  54          * to match the DMA mapping length, so we need to differentiate those.
  55          */
  56         if (tx_swbd->is_dma_page)
  57                 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
  58                                tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
  59                                tx_swbd->dir);
  60         else
  61                 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
  62                                  tx_swbd->len, tx_swbd->dir);
  63         tx_swbd->dma = 0;
  64 }
  65
  66 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
  67                                 struct enetc_tx_swbd *tx_swbd)
  68 {
  69         struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
  70         struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
  71
  72         if (tx_swbd->dma)
  73                 enetc_unmap_tx_buff(tx_ring, tx_swbd);
  74
  75         if (xdp_frame) {
  76                 xdp_return_frame(tx_swbd->xdp_frame);
  77                 tx_swbd->xdp_frame = NULL;
  78         } else if (skb) {
  79                 dev_kfree_skb_any(skb);
  80                 tx_swbd->skb = NULL;
  81         }
  82 }
  83
  84 /* Let H/W know BD ring has been updated */
  85 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
  86 {
  87         /* includes wmb() */
  88         enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
  89 }
  90
  91 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
  92                            u8 *msgtype, u8 *twostep,
  93                            u16 *correction_offset, u16 *body_offset)
  94 {
  95         unsigned int ptp_class;
  96         struct ptp_header *hdr;
  97         unsigned int type;
  98         u8 *base;
  99
 100         ptp_class = ptp_classify_raw(skb);
 101         if (ptp_class == PTP_CLASS_NONE)
 102                 return -EINVAL;
 103
 104         hdr = ptp_parse_header(skb, ptp_class);
 105         if (!hdr)
 106                 return -EINVAL;
 107
 108         type = ptp_class & PTP_CLASS_PMASK;
 109         if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
 110                 *udp = 1;
 111         else
 112                 *udp = 0;
 113
 114         *msgtype = ptp_get_msgtype(hdr, ptp_class);
 115         *twostep = hdr->flag_field[0] & 0x2;
 116
 117         base = skb_mac_header(skb);
 118         *correction_offset = (u8 *)&hdr->correction - base;
 119         *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
 120
 121         return 0;
 122 }
 123
 124 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
 125 {
 126         bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
 127         struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
 128         struct enetc_hw *hw = &priv->si->hw;
 129         struct enetc_tx_swbd *tx_swbd;
 130         int len = skb_headlen(skb);
 131         union enetc_tx_bd temp_bd;
 132         u8 msgtype, twostep, udp;
 133         union enetc_tx_bd *txbd;
 134         u16 offset1, offset2;
 135         int i, count = 0;
 136         skb_frag_t *frag;
 137         unsigned int f;
 138         dma_addr_t dma;
 139         u8 flags = 0;
 140
 141         i = tx_ring->next_to_use;
 142         txbd = ENETC_TXBD(*tx_ring, i);
 143         prefetchw(txbd);
 144
 145         dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
 146         if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
 147                 goto dma_err;
 148
 149         temp_bd.addr = cpu_to_le64(dma);
 150         temp_bd.buf_len = cpu_to_le16(len);
 151         temp_bd.lstatus = 0;
 152
 153         tx_swbd = &tx_ring->tx_swbd[i];
 154         tx_swbd->dma = dma;
 155         tx_swbd->len = len;
 156         tx_swbd->is_dma_page = 0;
 157         tx_swbd->dir = DMA_TO_DEVICE;
 158         count++;
 159
 160         do_vlan = skb_vlan_tag_present(skb);
 161         if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 162                 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
 163                                     &offset2) ||
 164                     msgtype != PTP_MSGTYPE_SYNC || twostep)
 165                         WARN_ONCE(1, "Bad packet for one-step timestamping\n");
 166                 else
 167                         do_onestep_tstamp = true;
 168         } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
 169                 do_twostep_tstamp = true;
 170         }
 171
 172         tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
 173         tx_swbd->check_wb = tx_swbd->do_twostep_tstamp;
 174
 175         if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
 176                 flags |= ENETC_TXBD_FLAGS_EX;
 177
 178         if (tx_ring->tsd_enable)
 179                 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
 180
 181         /* first BD needs frm_len and offload flags set */
 182         temp_bd.frm_len = cpu_to_le16(skb->len);
 183         temp_bd.flags = flags;
 184
 185         if (flags & ENETC_TXBD_FLAGS_TSE)
 186                 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
 187                                                           flags);
 188
 189         if (flags & ENETC_TXBD_FLAGS_EX) {
 190                 u8 e_flags = 0;
 191                 *txbd = temp_bd;
 192                 enetc_clear_tx_bd(&temp_bd);
 193
 194                 /* add extension BD for VLAN and/or timestamping */
 195                 flags = 0;
 196                 tx_swbd++;
 197                 txbd++;
 198                 i++;
 199                 if (unlikely(i == tx_ring->bd_count)) {
 200                         i = 0;
 201                         tx_swbd = tx_ring->tx_swbd;
 202                         txbd = ENETC_TXBD(*tx_ring, 0);
 203                 }
 204                 prefetchw(txbd);
 205
 206                 if (do_vlan) {
 207                         temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
 208                         temp_bd.ext.tpid = 0; /* < C-TAG */
 209                         e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
 210                 }
 211
 212                 if (do_onestep_tstamp) {
 213                         u32 lo, hi, val;
 214                         u64 sec, nsec;
 215                         u8 *data;
 216
 217                         lo = enetc_rd_hot(hw, ENETC_SICTR0);
 218                         hi = enetc_rd_hot(hw, ENETC_SICTR1);
 219                         sec = (u64)hi << 32 | lo;
 220                         nsec = do_div(sec, 1000000000);
 221
 222                         /* Configure extension BD */
 223                         temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
 224                         e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
 225
 226                         /* Update originTimestamp field of Sync packet
 227                          * - 48 bits seconds field
 228                          * - 32 bits nanseconds field
 229                          */
 230                         data = skb_mac_header(skb);
 231                         *(__be16 *)(data + offset2) =
 232                                 htons((sec >> 32) & 0xffff);
 233                         *(__be32 *)(data + offset2 + 2) =
 234                                 htonl(sec & 0xffffffff);
 235                         *(__be32 *)(data + offset2 + 6) = htonl(nsec);
 236
 237                         /* Configure single-step register */
 238                         val = ENETC_PM0_SINGLE_STEP_EN;
 239                         val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
 240                         if (udp)
 241                                 val |= ENETC_PM0_SINGLE_STEP_CH;
 242
 243                         enetc_port_wr(hw, ENETC_PM0_SINGLE_STEP, val);
 244                         enetc_port_wr(hw, ENETC_PM1_SINGLE_STEP, val);
 245                 } else if (do_twostep_tstamp) {
 246                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 247                         e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
 248                 }
 249
 250                 temp_bd.ext.e_flags = e_flags;
 251                 count++;
 252         }
 253
 254         frag = &skb_shinfo(skb)->frags[0];
 255         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
 256                 len = skb_frag_size(frag);
 257                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
 258                                        DMA_TO_DEVICE);
 259                 if (dma_mapping_error(tx_ring->dev, dma))
 260                         goto dma_err;
 261
 262                 *txbd = temp_bd;
 263                 enetc_clear_tx_bd(&temp_bd);
 264
 265                 flags = 0;
 266                 tx_swbd++;
 267                 txbd++;
 268                 i++;
 269                 if (unlikely(i == tx_ring->bd_count)) {
 270                         i = 0;
 271                         tx_swbd = tx_ring->tx_swbd;
 272                         txbd = ENETC_TXBD(*tx_ring, 0);
 273                 }
 274                 prefetchw(txbd);
 275
 276                 temp_bd.addr = cpu_to_le64(dma);
 277                 temp_bd.buf_len = cpu_to_le16(len);
 278
 279                 tx_swbd->dma = dma;
 280                 tx_swbd->len = len;
 281                 tx_swbd->is_dma_page = 1;
 282                 tx_swbd->dir = DMA_TO_DEVICE;
 283                 count++;
 284         }
 285
 286         /* last BD needs 'F' bit set */
 287         flags |= ENETC_TXBD_FLAGS_F;
 288         temp_bd.flags = flags;
 289         *txbd = temp_bd;
 290
 291         tx_ring->tx_swbd[i].is_eof = true;
 292         tx_ring->tx_swbd[i].skb = skb;
 293
 294         enetc_bdr_idx_inc(tx_ring, &i);
 295         tx_ring->next_to_use = i;
 296
 297         skb_tx_timestamp(skb);
 298
 299         enetc_update_tx_ring_tail(tx_ring);
 300
 301         return count;
 302
 303 dma_err:
 304         dev_err(tx_ring->dev, "DMA map error");
 305
 306         do {
 307                 tx_swbd = &tx_ring->tx_swbd[i];
 308                 enetc_free_tx_frame(tx_ring, tx_swbd);
 309                 if (i == 0)
 310                         i = tx_ring->bd_count;
 311                 i--;
 312         } while (count--);
 313
 314         return 0;
 315 }
 316
 317 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
 318                                     struct net_device *ndev)
 319 {
 320         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 321         struct enetc_bdr *tx_ring;
 322         int count;
 323
 324         /* Queue one-step Sync packet if already locked */
 325         if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 326                 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
 327                                           &priv->flags)) {
 328                         skb_queue_tail(&priv->tx_skbs, skb);
 329                         return NETDEV_TX_OK;
 330                 }
 331         }
 332
 333         tx_ring = priv->tx_ring[skb->queue_mapping];
 334
 335         if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
 336                 if (unlikely(skb_linearize(skb)))
 337                         goto drop_packet_err;
 338
 339         count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
 340         if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
 341                 netif_stop_subqueue(ndev, tx_ring->index);
 342                 return NETDEV_TX_BUSY;
 343         }
 344
 345         enetc_lock_mdio();
 346         count = enetc_map_tx_buffs(tx_ring, skb);
 347         enetc_unlock_mdio();
 348
 349         if (unlikely(!count))
 350                 goto drop_packet_err;
 351
 352         if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
 353                 netif_stop_subqueue(ndev, tx_ring->index);
 354
 355         return NETDEV_TX_OK;
 356
 357 drop_packet_err:
 358         dev_kfree_skb_any(skb);
 359         return NETDEV_TX_OK;
 360 }
 361
 362 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
 363 {
 364         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 365         u8 udp, msgtype, twostep;
 366         u16 offset1, offset2;
 367
 368         /* Mark tx timestamp type on skb->cb[0] if requires */
 369         if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
 370             (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
 371                 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
 372         } else {
 373                 skb->cb[0] = 0;
 374         }
 375
 376         /* Fall back to two-step timestamp if not one-step Sync packet */
 377         if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 378                 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
 379                                     &offset1, &offset2) ||
 380                     msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
 381                         skb->cb[0] = ENETC_F_TX_TSTAMP;
 382         }
 383
 384         return enetc_start_xmit(skb, ndev);
 385 }
 386
 387 static irqreturn_t enetc_msix(int irq, void *data)
 388 {
 389         struct enetc_int_vector *v = data;
 390         int i;
 391
 392         enetc_lock_mdio();
 393
 394         /* disable interrupts */
 395         enetc_wr_reg_hot(v->rbier, 0);
 396         enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
 397
 398         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
 399                 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
 400
 401         enetc_unlock_mdio();
 402
 403         napi_schedule(&v->napi);
 404
 405         return IRQ_HANDLED;
 406 }
 407
 408 static void enetc_rx_dim_work(struct work_struct *w)
 409 {
 410         struct dim *dim = container_of(w, struct dim, work);
 411         struct dim_cq_moder moder =
 412                 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
 413         struct enetc_int_vector *v =
 414                 container_of(dim, struct enetc_int_vector, rx_dim);
 415
 416         v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
 417         dim->state = DIM_START_MEASURE;
 418 }
 419
 420 static void enetc_rx_net_dim(struct enetc_int_vector *v)
 421 {
 422         struct dim_sample dim_sample;
 423
 424         v->comp_cnt++;
 425
 426         if (!v->rx_napi_work)
 427                 return;
 428
 429         dim_update_sample(v->comp_cnt,
 430                           v->rx_ring.stats.packets,
 431                           v->rx_ring.stats.bytes,
 432                           &dim_sample);
 433         net_dim(&v->rx_dim, dim_sample);
 434 }
 435
 436 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
 437 {
 438         int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
 439
 440         return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
 441 }
 442
 443 static bool enetc_page_reusable(struct page *page)
 444 {
 445         return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
 446 }
 447
 448 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
 449                              struct enetc_rx_swbd *old)
 450 {
 451         struct enetc_rx_swbd *new;
 452
 453         new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
 454
 455         /* next buf that may reuse a page */
 456         enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
 457
 458         /* copy page reference */
 459         *new = *old;
 460 }
 461
 462 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
 463                                 u64 *tstamp)
 464 {
 465         u32 lo, hi, tstamp_lo;
 466
 467         lo = enetc_rd_hot(hw, ENETC_SICTR0);
 468         hi = enetc_rd_hot(hw, ENETC_SICTR1);
 469         tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
 470         if (lo <= tstamp_lo)
 471                 hi -= 1;
 472         *tstamp = (u64)hi << 32 | tstamp_lo;
 473 }
 474
 475 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
 476 {
 477         struct skb_shared_hwtstamps shhwtstamps;
 478
 479         if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
 480                 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
 481                 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
 482                 skb_txtime_consumed(skb);
 483                 skb_tstamp_tx(skb, &shhwtstamps);
 484         }
 485 }
 486
 487 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
 488                                       struct enetc_tx_swbd *tx_swbd)
 489 {
 490         struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
 491         struct enetc_rx_swbd rx_swbd = {
 492                 .dma = tx_swbd->dma,
 493                 .page = tx_swbd->page,
 494                 .page_offset = tx_swbd->page_offset,
 495                 .dir = tx_swbd->dir,
 496                 .len = tx_swbd->len,
 497         };
 498         struct enetc_bdr *rx_ring;
 499
 500         rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
 501
 502         if (likely(enetc_swbd_unused(rx_ring))) {
 503                 enetc_reuse_page(rx_ring, &rx_swbd);
 504
 505                 /* sync for use by the device */
 506                 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
 507                                                  rx_swbd.page_offset,
 508                                                  ENETC_RXB_DMA_SIZE_XDP,
 509                                                  rx_swbd.dir);
 510
 511                 rx_ring->stats.recycles++;
 512         } else {
 513                 /* RX ring is already full, we need to unmap and free the
 514                  * page, since there's nothing useful we can do with it.
 515                  */
 516                 rx_ring->stats.recycle_failures++;
 517
 518                 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
 519                                rx_swbd.dir);
 520                 __free_page(rx_swbd.page);
 521         }
 522
 523         rx_ring->xdp.xdp_tx_in_flight--;
 524 }
 525
 526 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
 527 {
 528         struct net_device *ndev = tx_ring->ndev;
 529         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 530         int tx_frm_cnt = 0, tx_byte_cnt = 0;
 531         struct enetc_tx_swbd *tx_swbd;
 532         int i, bds_to_clean;
 533         bool do_twostep_tstamp;
 534         u64 tstamp = 0;
 535
 536         i = tx_ring->next_to_clean;
 537         tx_swbd = &tx_ring->tx_swbd[i];
 538
 539         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 540
 541         do_twostep_tstamp = false;
 542
 543         while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
 544                 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
 545                 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
 546                 bool is_eof = tx_swbd->is_eof;
 547
 548                 if (unlikely(tx_swbd->check_wb)) {
 549                         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 550                         union enetc_tx_bd *txbd;
 551
 552                         txbd = ENETC_TXBD(*tx_ring, i);
 553
 554                         if (txbd->flags & ENETC_TXBD_FLAGS_W &&
 555                             tx_swbd->do_twostep_tstamp) {
 556                                 enetc_get_tx_tstamp(&priv->si->hw, txbd,
 557                                                     &tstamp);
 558                                 do_twostep_tstamp = true;
 559                         }
 560                 }
 561
 562                 if (tx_swbd->is_xdp_tx)
 563                         enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
 564                 else if (likely(tx_swbd->dma))
 565                         enetc_unmap_tx_buff(tx_ring, tx_swbd);
 566
 567                 if (xdp_frame) {
 568                         xdp_return_frame(xdp_frame);
 569                 } else if (skb) {
 570                         if (unlikely(tx_swbd->skb->cb[0] &
 571                                      ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
 572                                 /* Start work to release lock for next one-step
 573                                  * timestamping packet. And send one skb in
 574                                  * tx_skbs queue if has.
 575                                  */
 576                                 schedule_work(&priv->tx_onestep_tstamp);
 577                         } else if (unlikely(do_twostep_tstamp)) {
 578                                 enetc_tstamp_tx(skb, tstamp);
 579                                 do_twostep_tstamp = false;
 580                         }
 581                         napi_consume_skb(skb, napi_budget);
 582                 }
 583
 584                 tx_byte_cnt += tx_swbd->len;
 585                 /* Scrub the swbd here so we don't have to do that
 586                  * when we reuse it during xmit
 587                  */
 588                 memset(tx_swbd, 0, sizeof(*tx_swbd));
 589
 590                 bds_to_clean--;
 591                 tx_swbd++;
 592                 i++;
 593                 if (unlikely(i == tx_ring->bd_count)) {
 594                         i = 0;
 595                         tx_swbd = tx_ring->tx_swbd;
 596                 }
 597
 598                 /* BD iteration loop end */
 599                 if (is_eof) {
 600                         tx_frm_cnt++;
 601                         /* re-arm interrupt source */
 602                         enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
 603                                          BIT(16 + tx_ring->index));
 604                 }
 605
 606                 if (unlikely(!bds_to_clean))
 607                         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 608         }
 609
 610         tx_ring->next_to_clean = i;
 611         tx_ring->stats.packets += tx_frm_cnt;
 612         tx_ring->stats.bytes += tx_byte_cnt;
 613
 614         if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
 615                      __netif_subqueue_stopped(ndev, tx_ring->index) &&
 616                      (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
 617                 netif_wake_subqueue(ndev, tx_ring->index);
 618         }
 619
 620         return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
 621 }
 622
 623 static bool enetc_new_page(struct enetc_bdr *rx_ring,
 624                            struct enetc_rx_swbd *rx_swbd)
 625 {
 626         bool xdp = !!(rx_ring->xdp.prog);
 627         struct page *page;
 628         dma_addr_t addr;
 629
 630         page = dev_alloc_page();
 631         if (unlikely(!page))
 632                 return false;
 633
 634         /* For XDP_TX, we forgo dma_unmap -> dma_map */
 635         rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 636
 637         addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
 638         if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
 639                 __free_page(page);
 640
 641                 return false;
 642         }
 643
 644         rx_swbd->dma = addr;
 645         rx_swbd->page = page;
 646         rx_swbd->page_offset = rx_ring->buffer_offset;
 647
 648         return true;
 649 }
 650
 651 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
 652 {
 653         struct enetc_rx_swbd *rx_swbd;
 654         union enetc_rx_bd *rxbd;
 655         int i, j;
 656
 657         i = rx_ring->next_to_use;
 658         rx_swbd = &rx_ring->rx_swbd[i];
 659         rxbd = enetc_rxbd(rx_ring, i);
 660
 661         for (j = 0; j < buff_cnt; j++) {
 662                 /* try reuse page */
 663                 if (unlikely(!rx_swbd->page)) {
 664                         if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
 665                                 rx_ring->stats.rx_alloc_errs++;
 666                                 break;
 667                         }
 668                 }
 669
 670                 /* update RxBD */
 671                 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
 672                                            rx_swbd->page_offset);
 673                 /* clear 'R" as well */
 674                 rxbd->r.lstatus = 0;
 675
 676                 enetc_rxbd_next(rx_ring, &rxbd, &i);
 677                 rx_swbd = &rx_ring->rx_swbd[i];
 678         }
 679
 680         if (likely(j)) {
 681                 rx_ring->next_to_alloc = i; /* keep track from page reuse */
 682                 rx_ring->next_to_use = i;
 683
 684                 /* update ENETC's consumer index */
 685                 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
 686         }
 687
 688         return j;
 689 }
 690
 691 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 692 static void enetc_get_rx_tstamp(struct net_device *ndev,
 693                                 union enetc_rx_bd *rxbd,
 694                                 struct sk_buff *skb)
 695 {
 696         struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
 697         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 698         struct enetc_hw *hw = &priv->si->hw;
 699         u32 lo, hi, tstamp_lo;
 700         u64 tstamp;
 701
 702         if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
 703                 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
 704                 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
 705                 rxbd = enetc_rxbd_ext(rxbd);
 706                 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
 707                 if (lo <= tstamp_lo)
 708                         hi -= 1;
 709
 710                 tstamp = (u64)hi << 32 | tstamp_lo;
 711                 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
 712                 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
 713         }
 714 }
 715 #endif
 716
 717 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
 718                                union enetc_rx_bd *rxbd, struct sk_buff *skb)
 719 {
 720         struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
 721
 722         /* TODO: hashing */
 723         if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
 724                 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
 725
 726                 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
 727                 skb->ip_summed = CHECKSUM_COMPLETE;
 728         }
 729
 730         if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
 731                 __be16 tpid = 0;
 732
 733                 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
 734                 case 0:
 735                         tpid = htons(ETH_P_8021Q);
 736                         break;
 737                 case 1:
 738                         tpid = htons(ETH_P_8021AD);
 739                         break;
 740                 case 2:
 741                         tpid = htons(enetc_port_rd(&priv->si->hw,
 742                                                    ENETC_PCVLANR1));
 743                         break;
 744                 case 3:
 745                         tpid = htons(enetc_port_rd(&priv->si->hw,
 746                                                    ENETC_PCVLANR2));
 747                         break;
 748                 default:
 749                         break;
 750                 }
 751
 752                 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
 753         }
 754
 755 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 756         if (priv->active_offloads & ENETC_F_RX_TSTAMP)
 757                 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
 758 #endif
 759 }
 760
 761 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
 762  * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
 763  * mapped buffers.
 764  */
 765 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
 766                                                int i, u16 size)
 767 {
 768         struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
 769
 770         dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
 771                                       rx_swbd->page_offset,
 772                                       size, rx_swbd->dir);
 773         return rx_swbd;
 774 }
 775
 776 /* Reuse the current page without performing half-page buffer flipping */
 777 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
 778                               struct enetc_rx_swbd *rx_swbd)
 779 {
 780         size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
 781
 782         enetc_reuse_page(rx_ring, rx_swbd);
 783
 784         dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
 785                                          rx_swbd->page_offset,
 786                                          buffer_size, rx_swbd->dir);
 787
 788         rx_swbd->page = NULL;
 789 }
 790
 791 /* Reuse the current page by performing half-page buffer flipping */
 792 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
 793                                struct enetc_rx_swbd *rx_swbd)
 794 {
 795         if (likely(enetc_page_reusable(rx_swbd->page))) {
 796                 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
 797                 page_ref_inc(rx_swbd->page);
 798
 799                 enetc_put_rx_buff(rx_ring, rx_swbd);
 800         } else {
 801                 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
 802                                rx_swbd->dir);
 803                 rx_swbd->page = NULL;
 804         }
 805 }
 806
 807 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
 808                                                 int i, u16 size)
 809 {
 810         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
 811         struct sk_buff *skb;
 812         void *ba;
 813
 814         ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
 815         skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
 816         if (unlikely(!skb)) {
 817                 rx_ring->stats.rx_alloc_errs++;
 818                 return NULL;
 819         }
 820
 821         skb_reserve(skb, rx_ring->buffer_offset);
 822         __skb_put(skb, size);
 823
 824         enetc_flip_rx_buff(rx_ring, rx_swbd);
 825
 826         return skb;
 827 }
 828
 829 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
 830                                      u16 size, struct sk_buff *skb)
 831 {
 832         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
 833
 834         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
 835                         rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
 836
 837         enetc_flip_rx_buff(rx_ring, rx_swbd);
 838 }
 839
 840 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
 841                                               u32 bd_status,
 842                                               union enetc_rx_bd **rxbd, int *i)
 843 {
 844         if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
 845                 return false;
 846
 847         enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
 848         enetc_rxbd_next(rx_ring, rxbd, i);
 849
 850         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
 851                 dma_rmb();
 852                 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
 853
 854                 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
 855                 enetc_rxbd_next(rx_ring, rxbd, i);
 856         }
 857
 858         rx_ring->ndev->stats.rx_dropped++;
 859         rx_ring->ndev->stats.rx_errors++;
 860
 861         return true;
 862 }
 863
 864 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
 865                                        u32 bd_status, union enetc_rx_bd **rxbd,
 866                                        int *i, int *cleaned_cnt, int buffer_size)
 867 {
 868         struct sk_buff *skb;
 869         u16 size;
 870
 871         size = le16_to_cpu((*rxbd)->r.buf_len);
 872         skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
 873         if (!skb)
 874                 return NULL;
 875
 876         enetc_get_offloads(rx_ring, *rxbd, skb);
 877
 878         (*cleaned_cnt)++;
 879
 880         enetc_rxbd_next(rx_ring, rxbd, i);
 881
 882         /* not last BD in frame? */
 883         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
 884                 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
 885                 size = buffer_size;
 886
 887                 if (bd_status & ENETC_RXBD_LSTATUS_F) {
 888                         dma_rmb();
 889                         size = le16_to_cpu((*rxbd)->r.buf_len);
 890                 }
 891
 892                 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
 893
 894                 (*cleaned_cnt)++;
 895
 896                 enetc_rxbd_next(rx_ring, rxbd, i);
 897         }
 898
 899         skb_record_rx_queue(skb, rx_ring->index);
 900         skb->protocol = eth_type_trans(skb, rx_ring->ndev);
 901
 902         return skb;
 903 }
 904
 905 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
 906
 907 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
 908                                struct napi_struct *napi, int work_limit)
 909 {
 910         int rx_frm_cnt = 0, rx_byte_cnt = 0;
 911         int cleaned_cnt, i;
 912
 913         cleaned_cnt = enetc_bd_unused(rx_ring);
 914         /* next descriptor to process */
 915         i = rx_ring->next_to_clean;
 916
 917         while (likely(rx_frm_cnt < work_limit)) {
 918                 union enetc_rx_bd *rxbd;
 919                 struct sk_buff *skb;
 920                 u32 bd_status;
 921
 922                 if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
 923                         cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
 924                                                             cleaned_cnt);
 925
 926                 rxbd = enetc_rxbd(rx_ring, i);
 927                 bd_status = le32_to_cpu(rxbd->r.lstatus);
 928                 if (!bd_status)
 929                         break;
 930
 931                 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
 932                 dma_rmb(); /* for reading other rxbd fields */
 933
 934                 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
 935                                                       &rxbd, &i))
 936                         break;
 937
 938                 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
 939                                       &cleaned_cnt, ENETC_RXB_DMA_SIZE);
 940                 if (!skb)
 941                         break;
 942
 943                 rx_byte_cnt += skb->len;
 944                 rx_frm_cnt++;
 945
 946                 napi_gro_receive(napi, skb);
 947         }
 948
 949         rx_ring->next_to_clean = i;
 950
 951         rx_ring->stats.packets += rx_frm_cnt;
 952         rx_ring->stats.bytes += rx_byte_cnt;
 953
 954         return rx_frm_cnt;
 955 }
 956
 957 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
 958                                   struct enetc_tx_swbd *tx_swbd,
 959                                   int frm_len)
 960 {
 961         union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
 962
 963         prefetchw(txbd);
 964
 965         enetc_clear_tx_bd(txbd);
 966         txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
 967         txbd->buf_len = cpu_to_le16(tx_swbd->len);
 968         txbd->frm_len = cpu_to_le16(frm_len);
 969
 970         memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
 971 }
 972
 973 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
 974  * descriptors.
 975  */
 976 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
 977                          struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
 978 {
 979         struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
 980         int i, k, frm_len = tmp_tx_swbd->len;
 981
 982         if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
 983                 return false;
 984
 985         while (unlikely(!tmp_tx_swbd->is_eof)) {
 986                 tmp_tx_swbd++;
 987                 frm_len += tmp_tx_swbd->len;
 988         }
 989
 990         i = tx_ring->next_to_use;
 991
 992         for (k = 0; k < num_tx_swbd; k++) {
 993                 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
 994
 995                 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
 996
 997                 /* last BD needs 'F' bit set */
 998                 if (xdp_tx_swbd->is_eof) {
 999                         union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1000
1001                         txbd->flags = ENETC_TXBD_FLAGS_F;
1002                 }
1003
1004                 enetc_bdr_idx_inc(tx_ring, &i);
1005         }
1006
1007         tx_ring->next_to_use = i;
1008
1009         return true;
1010 }
1011
1012 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1013                                           struct enetc_tx_swbd *xdp_tx_arr,
1014                                           struct xdp_frame *xdp_frame)
1015 {
1016         struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1017         struct skb_shared_info *shinfo;
1018         void *data = xdp_frame->data;
1019         int len = xdp_frame->len;
1020         skb_frag_t *frag;
1021         dma_addr_t dma;
1022         unsigned int f;
1023         int n = 0;
1024
1025         dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1026         if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1027                 netdev_err(tx_ring->ndev, "DMA map error\n");
1028                 return -1;
1029         }
1030
1031         xdp_tx_swbd->dma = dma;
1032         xdp_tx_swbd->dir = DMA_TO_DEVICE;
1033         xdp_tx_swbd->len = len;
1034         xdp_tx_swbd->is_xdp_redirect = true;
1035         xdp_tx_swbd->is_eof = false;
1036         xdp_tx_swbd->xdp_frame = NULL;
1037
1038         n++;
1039         xdp_tx_swbd = &xdp_tx_arr[n];
1040
1041         shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1042
1043         for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1044              f++, frag++) {
1045                 data = skb_frag_address(frag);
1046                 len = skb_frag_size(frag);
1047
1048                 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1049                 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1050                         /* Undo the DMA mapping for all fragments */
1051                         while (--n >= 0)
1052                                 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1053
1054                         netdev_err(tx_ring->ndev, "DMA map error\n");
1055                         return -1;
1056                 }
1057
1058                 xdp_tx_swbd->dma = dma;
1059                 xdp_tx_swbd->dir = DMA_TO_DEVICE;
1060                 xdp_tx_swbd->len = len;
1061                 xdp_tx_swbd->is_xdp_redirect = true;
1062                 xdp_tx_swbd->is_eof = false;
1063                 xdp_tx_swbd->xdp_frame = NULL;
1064
1065                 n++;
1066                 xdp_tx_swbd = &xdp_tx_arr[n];
1067         }
1068
1069         xdp_tx_arr[n - 1].is_eof = true;
1070         xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1071
1072         return n;
1073 }
1074
1075 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1076                    struct xdp_frame **frames, u32 flags)
1077 {
1078         struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1079         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1080         struct enetc_bdr *tx_ring;
1081         int xdp_tx_bd_cnt, i, k;
1082         int xdp_tx_frm_cnt = 0;
1083
1084         enetc_lock_mdio();
1085
1086         tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1087
1088         prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1089
1090         for (k = 0; k < num_frames; k++) {
1091                 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1092                                                                xdp_redirect_arr,
1093                                                                frames[k]);
1094                 if (unlikely(xdp_tx_bd_cnt < 0))
1095                         break;
1096
1097                 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1098                                            xdp_tx_bd_cnt))) {
1099                         for (i = 0; i < xdp_tx_bd_cnt; i++)
1100                                 enetc_unmap_tx_buff(tx_ring,
1101                                                     &xdp_redirect_arr[i]);
1102                         tx_ring->stats.xdp_tx_drops++;
1103                         break;
1104                 }
1105
1106                 xdp_tx_frm_cnt++;
1107         }
1108
1109         if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1110                 enetc_update_tx_ring_tail(tx_ring);
1111
1112         tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1113
1114         enetc_unlock_mdio();
1115
1116         return xdp_tx_frm_cnt;
1117 }
1118
1119 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1120                                      struct xdp_buff *xdp_buff, u16 size)
1121 {
1122         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1123         void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1124         struct skb_shared_info *shinfo;
1125
1126         /* To be used for XDP_TX */
1127         rx_swbd->len = size;
1128
1129         xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1130                          rx_ring->buffer_offset, size, false);
1131
1132         shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1133         shinfo->nr_frags = 0;
1134 }
1135
1136 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1137                                      u16 size, struct xdp_buff *xdp_buff)
1138 {
1139         struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1140         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1141         skb_frag_t *frag = &shinfo->frags[shinfo->nr_frags];
1142
1143         /* To be used for XDP_TX */
1144         rx_swbd->len = size;
1145
1146         skb_frag_off_set(frag, rx_swbd->page_offset);
1147         skb_frag_size_set(frag, size);
1148         __skb_frag_set_page(frag, rx_swbd->page);
1149
1150         shinfo->nr_frags++;
1151 }
1152
1153 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1154                                  union enetc_rx_bd **rxbd, int *i,
1155                                  int *cleaned_cnt, struct xdp_buff *xdp_buff)
1156 {
1157         u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1158
1159         xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1160
1161         enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1162         (*cleaned_cnt)++;
1163         enetc_rxbd_next(rx_ring, rxbd, i);
1164
1165         /* not last BD in frame? */
1166         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1167                 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1168                 size = ENETC_RXB_DMA_SIZE_XDP;
1169
1170                 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1171                         dma_rmb();
1172                         size = le16_to_cpu((*rxbd)->r.buf_len);
1173                 }
1174
1175                 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1176                 (*cleaned_cnt)++;
1177                 enetc_rxbd_next(rx_ring, rxbd, i);
1178         }
1179 }
1180
1181 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1182  * recycled back into the RX ring in enetc_clean_tx_ring.
1183  */
1184 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1185                                         struct enetc_bdr *rx_ring,
1186                                         int rx_ring_first, int rx_ring_last)
1187 {
1188         int n = 0;
1189
1190         for (; rx_ring_first != rx_ring_last;
1191              n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1192                 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1193                 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1194
1195                 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1196                 tx_swbd->dma = rx_swbd->dma;
1197                 tx_swbd->dir = rx_swbd->dir;
1198                 tx_swbd->page = rx_swbd->page;
1199                 tx_swbd->page_offset = rx_swbd->page_offset;
1200                 tx_swbd->len = rx_swbd->len;
1201                 tx_swbd->is_dma_page = true;
1202                 tx_swbd->is_xdp_tx = true;
1203                 tx_swbd->is_eof = false;
1204         }
1205
1206         /* We rely on caller providing an rx_ring_last > rx_ring_first */
1207         xdp_tx_arr[n - 1].is_eof = true;
1208
1209         return n;
1210 }
1211
1212 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1213                            int rx_ring_last)
1214 {
1215         while (rx_ring_first != rx_ring_last) {
1216                 enetc_put_rx_buff(rx_ring,
1217                                   &rx_ring->rx_swbd[rx_ring_first]);
1218                 enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1219         }
1220         rx_ring->stats.xdp_drops++;
1221 }
1222
1223 static void enetc_xdp_free(struct enetc_bdr *rx_ring, int rx_ring_first,
1224                            int rx_ring_last)
1225 {
1226         while (rx_ring_first != rx_ring_last) {
1227                 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1228
1229                 if (rx_swbd->page) {
1230                         dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1231                                        rx_swbd->dir);
1232                         __free_page(rx_swbd->page);
1233                         rx_swbd->page = NULL;
1234                 }
1235                 enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1236         }
1237         rx_ring->stats.xdp_redirect_failures++;
1238 }
1239
1240 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1241                                    struct napi_struct *napi, int work_limit,
1242                                    struct bpf_prog *prog)
1243 {
1244         int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1245         struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1246         struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1247         int rx_frm_cnt = 0, rx_byte_cnt = 0;
1248         struct enetc_bdr *tx_ring;
1249         int cleaned_cnt, i;
1250         u32 xdp_act;
1251
1252         cleaned_cnt = enetc_bd_unused(rx_ring);
1253         /* next descriptor to process */
1254         i = rx_ring->next_to_clean;
1255
1256         while (likely(rx_frm_cnt < work_limit)) {
1257                 union enetc_rx_bd *rxbd, *orig_rxbd;
1258                 int orig_i, orig_cleaned_cnt;
1259                 struct xdp_buff xdp_buff;
1260                 struct sk_buff *skb;
1261                 int tmp_orig_i, err;
1262                 u32 bd_status;
1263
1264                 rxbd = enetc_rxbd(rx_ring, i);
1265                 bd_status = le32_to_cpu(rxbd->r.lstatus);
1266                 if (!bd_status)
1267                         break;
1268
1269                 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1270                 dma_rmb(); /* for reading other rxbd fields */
1271
1272                 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1273                                                       &rxbd, &i))
1274                         break;
1275
1276                 orig_rxbd = rxbd;
1277                 orig_cleaned_cnt = cleaned_cnt;
1278                 orig_i = i;
1279
1280                 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1281                                      &cleaned_cnt, &xdp_buff);
1282
1283                 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1284
1285                 switch (xdp_act) {
1286                 default:
1287                         bpf_warn_invalid_xdp_action(xdp_act);
1288                         fallthrough;
1289                 case XDP_ABORTED:
1290                         trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1291                         fallthrough;
1292                 case XDP_DROP:
1293                         enetc_xdp_drop(rx_ring, orig_i, i);
1294                         break;
1295                 case XDP_PASS:
1296                         rxbd = orig_rxbd;
1297                         cleaned_cnt = orig_cleaned_cnt;
1298                         i = orig_i;
1299
1300                         skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1301                                               &i, &cleaned_cnt,
1302                                               ENETC_RXB_DMA_SIZE_XDP);
1303                         if (unlikely(!skb))
1304                                 goto out;
1305
1306                         napi_gro_receive(napi, skb);
1307                         break;
1308                 case XDP_TX:
1309                         tx_ring = priv->xdp_tx_ring[rx_ring->index];
1310                         xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1311                                                                      rx_ring,
1312                                                                      orig_i, i);
1313
1314                         if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1315                                 enetc_xdp_drop(rx_ring, orig_i, i);
1316                                 tx_ring->stats.xdp_tx_drops++;
1317                         } else {
1318                                 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
1319                                 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1320                                 xdp_tx_frm_cnt++;
1321                                 /* The XDP_TX enqueue was successful, so we
1322                                  * need to scrub the RX software BDs because
1323                                  * the ownership of the buffers no longer
1324                                  * belongs to the RX ring, and we must prevent
1325                                  * enetc_refill_rx_ring() from reusing
1326                                  * rx_swbd->page.
1327                                  */
1328                                 while (orig_i != i) {
1329                                         rx_ring->rx_swbd[orig_i].page = NULL;
1330                                         enetc_bdr_idx_inc(rx_ring, &orig_i);
1331                                 }
1332                         }
1333                         break;
1334                 case XDP_REDIRECT:
1335                         /* xdp_return_frame does not support S/G in the sense
1336                          * that it leaks the fragments (__xdp_return should not
1337                          * call page_frag_free only for the initial buffer).
1338                          * Until XDP_REDIRECT gains support for S/G let's keep
1339                          * the code structure in place, but dead. We drop the
1340                          * S/G frames ourselves to avoid memory leaks which
1341                          * would otherwise leave the kernel OOM.
1342                          */
1343                         if (unlikely(cleaned_cnt - orig_cleaned_cnt != 1)) {
1344                                 enetc_xdp_drop(rx_ring, orig_i, i);
1345                                 rx_ring->stats.xdp_redirect_sg++;
1346                                 break;
1347                         }
1348
1349                         tmp_orig_i = orig_i;
1350
1351                         while (orig_i != i) {
1352                                 enetc_flip_rx_buff(rx_ring,
1353                                                    &rx_ring->rx_swbd[orig_i]);
1354                                 enetc_bdr_idx_inc(rx_ring, &orig_i);
1355                         }
1356
1357                         err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1358                         if (unlikely(err)) {
1359                                 enetc_xdp_free(rx_ring, tmp_orig_i, i);
1360                         } else {
1361                                 xdp_redirect_frm_cnt++;
1362                                 rx_ring->stats.xdp_redirect++;
1363                         }
1364                 }
1365
1366                 rx_frm_cnt++;
1367         }
1368
1369 out:
1370         rx_ring->next_to_clean = i;
1371
1372         rx_ring->stats.packets += rx_frm_cnt;
1373         rx_ring->stats.bytes += rx_byte_cnt;
1374
1375         if (xdp_redirect_frm_cnt)
1376                 xdp_do_flush_map();
1377
1378         if (xdp_tx_frm_cnt)
1379                 enetc_update_tx_ring_tail(tx_ring);
1380
1381         if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
1382                 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
1383                                      rx_ring->xdp.xdp_tx_in_flight);
1384
1385         return rx_frm_cnt;
1386 }
1387
1388 static int enetc_poll(struct napi_struct *napi, int budget)
1389 {
1390         struct enetc_int_vector
1391                 *v = container_of(napi, struct enetc_int_vector, napi);
1392         struct enetc_bdr *rx_ring = &v->rx_ring;
1393         struct bpf_prog *prog;
1394         bool complete = true;
1395         int work_done;
1396         int i;
1397
1398         enetc_lock_mdio();
1399
1400         for (i = 0; i < v->count_tx_rings; i++)
1401                 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
1402                         complete = false;
1403
1404         prog = rx_ring->xdp.prog;
1405         if (prog)
1406                 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1407         else
1408                 work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
1409         if (work_done == budget)
1410                 complete = false;
1411         if (work_done)
1412                 v->rx_napi_work = true;
1413
1414         if (!complete) {
1415                 enetc_unlock_mdio();
1416                 return budget;
1417         }
1418
1419         napi_complete_done(napi, work_done);
1420
1421         if (likely(v->rx_dim_en))
1422                 enetc_rx_net_dim(v);
1423
1424         v->rx_napi_work = false;
1425
1426         /* enable interrupts */
1427         enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
1428
1429         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1430                 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
1431                                  ENETC_TBIER_TXTIE);
1432
1433         enetc_unlock_mdio();
1434
1435         return work_done;
1436 }
1437
1438 /* Probing and Init */
1439 #define ENETC_MAX_RFS_SIZE 64
1440 void enetc_get_si_caps(struct enetc_si *si)
1441 {
1442         struct enetc_hw *hw = &si->hw;
1443         u32 val;
1444
1445         /* find out how many of various resources we have to work with */
1446         val = enetc_rd(hw, ENETC_SICAPR0);
1447         si->num_rx_rings = (val >> 16) & 0xff;
1448         si->num_tx_rings = val & 0xff;
1449
1450         val = enetc_rd(hw, ENETC_SIRFSCAPR);
1451         si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1452         si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1453
1454         si->num_rss = 0;
1455         val = enetc_rd(hw, ENETC_SIPCAPR0);
1456         if (val & ENETC_SIPCAPR0_RSS) {
1457                 u32 rss;
1458
1459                 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
1460                 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1461         }
1462
1463         if (val & ENETC_SIPCAPR0_QBV)
1464                 si->hw_features |= ENETC_SI_F_QBV;
1465
1466         if (val & ENETC_SIPCAPR0_PSFP)
1467                 si->hw_features |= ENETC_SI_F_PSFP;
1468 }
1469
1470 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
1471 {
1472         r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
1473                                         &r->bd_dma_base, GFP_KERNEL);
1474         if (!r->bd_base)
1475                 return -ENOMEM;
1476
1477         /* h/w requires 128B alignment */
1478         if (!IS_ALIGNED(r->bd_dma_base, 128)) {
1479                 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
1480                                   r->bd_dma_base);
1481                 return -EINVAL;
1482         }
1483
1484         return 0;
1485 }
1486
1487 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
1488 {
1489         int err;
1490
1491         txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
1492         if (!txr->tx_swbd)
1493                 return -ENOMEM;
1494
1495         err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
1496         if (err) {
1497                 vfree(txr->tx_swbd);
1498                 return err;
1499         }
1500
1501         txr->next_to_clean = 0;
1502         txr->next_to_use = 0;
1503
1504         return 0;
1505 }
1506
1507 static void enetc_free_txbdr(struct enetc_bdr *txr)
1508 {
1509         int size, i;
1510
1511         for (i = 0; i < txr->bd_count; i++)
1512                 enetc_free_tx_frame(txr, &txr->tx_swbd[i]);
1513
1514         size = txr->bd_count * sizeof(union enetc_tx_bd);
1515
1516         dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
1517         txr->bd_base = NULL;
1518
1519         vfree(txr->tx_swbd);
1520         txr->tx_swbd = NULL;
1521 }
1522
1523 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1524 {
1525         int i, err;
1526
1527         for (i = 0; i < priv->num_tx_rings; i++) {
1528                 err = enetc_alloc_txbdr(priv->tx_ring[i]);
1529
1530                 if (err)
1531                         goto fail;
1532         }
1533
1534         return 0;
1535
1536 fail:
1537         while (i-- > 0)
1538                 enetc_free_txbdr(priv->tx_ring[i]);
1539
1540         return err;
1541 }
1542
1543 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
1544 {
1545         int i;
1546
1547         for (i = 0; i < priv->num_tx_rings; i++)
1548                 enetc_free_txbdr(priv->tx_ring[i]);
1549 }
1550
1551 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
1552 {
1553         size_t size = sizeof(union enetc_rx_bd);
1554         int err;
1555
1556         rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
1557         if (!rxr->rx_swbd)
1558                 return -ENOMEM;
1559
1560         if (extended)
1561                 size *= 2;
1562
1563         err = enetc_dma_alloc_bdr(rxr, size);
1564         if (err) {
1565                 vfree(rxr->rx_swbd);
1566                 return err;
1567         }
1568
1569         rxr->next_to_clean = 0;
1570         rxr->next_to_use = 0;
1571         rxr->next_to_alloc = 0;
1572         rxr->ext_en = extended;
1573
1574         return 0;
1575 }
1576
1577 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
1578 {
1579         int size;
1580
1581         size = rxr->bd_count * sizeof(union enetc_rx_bd);
1582
1583         dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
1584         rxr->bd_base = NULL;
1585
1586         vfree(rxr->rx_swbd);
1587         rxr->rx_swbd = NULL;
1588 }
1589
1590 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
1591 {
1592         bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
1593         int i, err;
1594
1595         for (i = 0; i < priv->num_rx_rings; i++) {
1596                 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
1597
1598                 if (err)
1599                         goto fail;
1600         }
1601
1602         return 0;
1603
1604 fail:
1605         while (i-- > 0)
1606                 enetc_free_rxbdr(priv->rx_ring[i]);
1607
1608         return err;
1609 }
1610
1611 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
1612 {
1613         int i;
1614
1615         for (i = 0; i < priv->num_rx_rings; i++)
1616                 enetc_free_rxbdr(priv->rx_ring[i]);
1617 }
1618
1619 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
1620 {
1621         int i;
1622
1623         if (!tx_ring->tx_swbd)
1624                 return;
1625
1626         for (i = 0; i < tx_ring->bd_count; i++) {
1627                 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
1628
1629                 enetc_free_tx_frame(tx_ring, tx_swbd);
1630         }
1631
1632         tx_ring->next_to_clean = 0;
1633         tx_ring->next_to_use = 0;
1634 }
1635
1636 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
1637 {
1638         int i;
1639
1640         if (!rx_ring->rx_swbd)
1641                 return;
1642
1643         for (i = 0; i < rx_ring->bd_count; i++) {
1644                 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1645
1646                 if (!rx_swbd->page)
1647                         continue;
1648
1649                 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1650                                rx_swbd->dir);
1651                 __free_page(rx_swbd->page);
1652                 rx_swbd->page = NULL;
1653         }
1654
1655         rx_ring->next_to_clean = 0;
1656         rx_ring->next_to_use = 0;
1657         rx_ring->next_to_alloc = 0;
1658 }
1659
1660 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
1661 {
1662         int i;
1663
1664         for (i = 0; i < priv->num_rx_rings; i++)
1665                 enetc_free_rx_ring(priv->rx_ring[i]);
1666
1667         for (i = 0; i < priv->num_tx_rings; i++)
1668                 enetc_free_tx_ring(priv->tx_ring[i]);
1669 }
1670
1671 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1672 {
1673         int *rss_table;
1674         int i;
1675
1676         rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1677         if (!rss_table)
1678                 return -ENOMEM;
1679
1680         /* Set up RSS table defaults */
1681         for (i = 0; i < si->num_rss; i++)
1682                 rss_table[i] = i % num_groups;
1683
1684         enetc_set_rss_table(si, rss_table, si->num_rss);
1685
1686         kfree(rss_table);
1687
1688         return 0;
1689 }
1690
1691 int enetc_configure_si(struct enetc_ndev_priv *priv)
1692 {
1693         struct enetc_si *si = priv->si;
1694         struct enetc_hw *hw = &si->hw;
1695         int err;
1696
1697         /* set SI cache attributes */
1698         enetc_wr(hw, ENETC_SICAR0,
1699                  ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1700         enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1701         /* enable SI */
1702         enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1703
1704         if (si->num_rss) {
1705                 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1706                 if (err)
1707                         return err;
1708         }
1709
1710         return 0;
1711 }
1712
1713 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1714 {
1715         struct enetc_si *si = priv->si;
1716         int cpus = num_online_cpus();
1717
1718         priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
1719         priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
1720
1721         /* Enable all available TX rings in order to configure as many
1722          * priorities as possible, when needed.
1723          * TODO: Make # of TX rings run-time configurable
1724          */
1725         priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1726         priv->num_tx_rings = si->num_tx_rings;
1727         priv->bdr_int_num = cpus;
1728         priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
1729         priv->tx_ictt = ENETC_TXIC_TIMETHR;
1730 }
1731
1732 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1733 {
1734         struct enetc_si *si = priv->si;
1735
1736         priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1737                                   GFP_KERNEL);
1738         if (!priv->cls_rules)
1739                 return -ENOMEM;
1740
1741         return 0;
1742 }
1743
1744 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1745 {
1746         kfree(priv->cls_rules);
1747 }
1748
1749 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1750 {
1751         int idx = tx_ring->index;
1752         u32 tbmr;
1753
1754         enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1755                        lower_32_bits(tx_ring->bd_dma_base));
1756
1757         enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1758                        upper_32_bits(tx_ring->bd_dma_base));
1759
1760         WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
1761         enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1762                        ENETC_RTBLENR_LEN(tx_ring->bd_count));
1763
1764         /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1765         tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1766         tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1767
1768         /* enable Tx ints by setting pkt thr to 1 */
1769         enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
1770
1771         tbmr = ENETC_TBMR_EN;
1772         if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1773                 tbmr |= ENETC_TBMR_VIH;
1774
1775         /* enable ring */
1776         enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1777
1778         tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1779         tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1780         tx_ring->idr = hw->reg + ENETC_SITXIDR;
1781 }
1782
1783 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1784 {
1785         int idx = rx_ring->index;
1786         u32 rbmr;
1787
1788         enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1789                        lower_32_bits(rx_ring->bd_dma_base));
1790
1791         enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1792                        upper_32_bits(rx_ring->bd_dma_base));
1793
1794         WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
1795         enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1796                        ENETC_RTBLENR_LEN(rx_ring->bd_count));
1797
1798         if (rx_ring->xdp.prog)
1799                 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
1800         else
1801                 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1802
1803         enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1804
1805         /* enable Rx ints by setting pkt thr to 1 */
1806         enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
1807
1808         rbmr = ENETC_RBMR_EN;
1809
1810         if (rx_ring->ext_en)
1811                 rbmr |= ENETC_RBMR_BDS;
1812
1813         if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1814                 rbmr |= ENETC_RBMR_VTE;
1815
1816         rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1817         rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1818
1819         enetc_lock_mdio();
1820         enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1821         enetc_unlock_mdio();
1822
1823         /* enable ring */
1824         enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1825 }
1826
1827 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1828 {
1829         int i;
1830
1831         for (i = 0; i < priv->num_tx_rings; i++)
1832                 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]);
1833
1834         for (i = 0; i < priv->num_rx_rings; i++)
1835                 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1836 }
1837
1838 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1839 {
1840         int idx = rx_ring->index;
1841
1842         /* disable EN bit on ring */
1843         enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1844 }
1845
1846 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1847 {
1848         int delay = 8, timeout = 100;
1849         int idx = tx_ring->index;
1850
1851         /* disable EN bit on ring */
1852         enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1853
1854         /* wait for busy to clear */
1855         while (delay < timeout &&
1856                enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1857                 msleep(delay);
1858                 delay *= 2;
1859         }
1860
1861         if (delay >= timeout)
1862                 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1863                             idx);
1864 }
1865
1866 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1867 {
1868         int i;
1869
1870         for (i = 0; i < priv->num_tx_rings; i++)
1871                 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]);
1872
1873         for (i = 0; i < priv->num_rx_rings; i++)
1874                 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1875
1876         udelay(1);
1877 }
1878
1879 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1880 {
1881         struct pci_dev *pdev = priv->si->pdev;
1882         cpumask_t cpu_mask;
1883         int i, j, err;
1884
1885         for (i = 0; i < priv->bdr_int_num; i++) {
1886                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1887                 struct enetc_int_vector *v = priv->int_vector[i];
1888                 int entry = ENETC_BDR_INT_BASE_IDX + i;
1889                 struct enetc_hw *hw = &priv->si->hw;
1890
1891                 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1892                          priv->ndev->name, i);
1893                 err = request_irq(irq, enetc_msix, 0, v->name, v);
1894                 if (err) {
1895                         dev_err(priv->dev, "request_irq() failed!\n");
1896                         goto irq_err;
1897                 }
1898                 disable_irq(irq);
1899
1900                 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1901                 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1902                 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
1903
1904                 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1905
1906                 for (j = 0; j < v->count_tx_rings; j++) {
1907                         int idx = v->tx_ring[j].index;
1908
1909                         enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1910                 }
1911                 cpumask_clear(&cpu_mask);
1912                 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
1913                 irq_set_affinity_hint(irq, &cpu_mask);
1914         }
1915
1916         return 0;
1917
1918 irq_err:
1919         while (i--) {
1920                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1921
1922                 irq_set_affinity_hint(irq, NULL);
1923                 free_irq(irq, priv->int_vector[i]);
1924         }
1925
1926         return err;
1927 }
1928
1929 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1930 {
1931         struct pci_dev *pdev = priv->si->pdev;
1932         int i;
1933
1934         for (i = 0; i < priv->bdr_int_num; i++) {
1935                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1936
1937                 irq_set_affinity_hint(irq, NULL);
1938                 free_irq(irq, priv->int_vector[i]);
1939         }
1940 }
1941
1942 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
1943 {
1944         struct enetc_hw *hw = &priv->si->hw;
1945         u32 icpt, ictt;
1946         int i;
1947
1948         /* enable Tx & Rx event indication */
1949         if (priv->ic_mode &
1950             (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
1951                 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
1952                 /* init to non-0 minimum, will be adjusted later */
1953                 ictt = 0x1;
1954         } else {
1955                 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
1956                 ictt = 0;
1957         }
1958
1959         for (i = 0; i < priv->num_rx_rings; i++) {
1960                 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
1961                 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
1962                 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
1963         }
1964
1965         if (priv->ic_mode & ENETC_IC_TX_MANUAL)
1966                 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
1967         else
1968                 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
1969
1970         for (i = 0; i < priv->num_tx_rings; i++) {
1971                 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
1972                 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
1973                 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
1974         }
1975 }
1976
1977 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
1978 {
1979         int i;
1980
1981         for (i = 0; i < priv->num_tx_rings; i++)
1982                 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0);
1983
1984         for (i = 0; i < priv->num_rx_rings; i++)
1985                 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0);
1986 }
1987
1988 static int enetc_phylink_connect(struct net_device *ndev)
1989 {
1990         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1991         struct ethtool_eee edata;
1992         int err;
1993
1994         if (!priv->phylink)
1995                 return 0; /* phy-less mode */
1996
1997         err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
1998         if (err) {
1999                 dev_err(&ndev->dev, "could not attach to PHY\n");
2000                 return err;
2001         }
2002
2003         /* disable EEE autoneg, until ENETC driver supports it */
2004         memset(&edata, 0, sizeof(struct ethtool_eee));
2005         phylink_ethtool_set_eee(priv->phylink, &edata);
2006
2007         return 0;
2008 }
2009
2010 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2011 {
2012         struct enetc_ndev_priv *priv;
2013         struct sk_buff *skb;
2014
2015         priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2016
2017         netif_tx_lock(priv->ndev);
2018
2019         clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2020         skb = skb_dequeue(&priv->tx_skbs);
2021         if (skb)
2022                 enetc_start_xmit(skb, priv->ndev);
2023
2024         netif_tx_unlock(priv->ndev);
2025 }
2026
2027 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2028 {
2029         INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2030         skb_queue_head_init(&priv->tx_skbs);
2031 }
2032
2033 void enetc_start(struct net_device *ndev)
2034 {
2035         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2036         int i;
2037
2038         enetc_setup_interrupts(priv);
2039
2040         for (i = 0; i < priv->bdr_int_num; i++) {
2041                 int irq = pci_irq_vector(priv->si->pdev,
2042                                          ENETC_BDR_INT_BASE_IDX + i);
2043
2044                 napi_enable(&priv->int_vector[i]->napi);
2045                 enable_irq(irq);
2046         }
2047
2048         if (priv->phylink)
2049                 phylink_start(priv->phylink);
2050         else
2051                 netif_carrier_on(ndev);
2052
2053         netif_tx_start_all_queues(ndev);
2054 }
2055
2056 int enetc_open(struct net_device *ndev)
2057 {
2058         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2059         int num_stack_tx_queues;
2060         int err;
2061
2062         err = enetc_setup_irqs(priv);
2063         if (err)
2064                 return err;
2065
2066         err = enetc_phylink_connect(ndev);
2067         if (err)
2068                 goto err_phy_connect;
2069
2070         err = enetc_alloc_tx_resources(priv);
2071         if (err)
2072                 goto err_alloc_tx;
2073
2074         err = enetc_alloc_rx_resources(priv);
2075         if (err)
2076                 goto err_alloc_rx;
2077
2078         num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2079
2080         err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2081         if (err)
2082                 goto err_set_queues;
2083
2084         err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
2085         if (err)
2086                 goto err_set_queues;
2087
2088         enetc_tx_onestep_tstamp_init(priv);
2089         enetc_setup_bdrs(priv);
2090         enetc_start(ndev);
2091
2092         return 0;
2093
2094 err_set_queues:
2095         enetc_free_rx_resources(priv);
2096 err_alloc_rx:
2097         enetc_free_tx_resources(priv);
2098 err_alloc_tx:
2099         if (priv->phylink)
2100                 phylink_disconnect_phy(priv->phylink);
2101 err_phy_connect:
2102         enetc_free_irqs(priv);
2103
2104         return err;
2105 }
2106
2107 void enetc_stop(struct net_device *ndev)
2108 {
2109         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2110         int i;
2111
2112         netif_tx_stop_all_queues(ndev);
2113
2114         for (i = 0; i < priv->bdr_int_num; i++) {
2115                 int irq = pci_irq_vector(priv->si->pdev,
2116                                          ENETC_BDR_INT_BASE_IDX + i);
2117
2118                 disable_irq(irq);
2119                 napi_synchronize(&priv->int_vector[i]->napi);
2120                 napi_disable(&priv->int_vector[i]->napi);
2121         }
2122
2123         if (priv->phylink)
2124                 phylink_stop(priv->phylink);
2125         else
2126                 netif_carrier_off(ndev);
2127
2128         enetc_clear_interrupts(priv);
2129 }
2130
2131 int enetc_close(struct net_device *ndev)
2132 {
2133         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2134
2135         enetc_stop(ndev);
2136         enetc_clear_bdrs(priv);
2137
2138         if (priv->phylink)
2139                 phylink_disconnect_phy(priv->phylink);
2140         enetc_free_rxtx_rings(priv);
2141         enetc_free_rx_resources(priv);
2142         enetc_free_tx_resources(priv);
2143         enetc_free_irqs(priv);
2144
2145         return 0;
2146 }
2147
2148 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
2149 {
2150         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2151         struct tc_mqprio_qopt *mqprio = type_data;
2152         struct enetc_bdr *tx_ring;
2153         int num_stack_tx_queues;
2154         u8 num_tc;
2155         int i;
2156
2157         num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2158         mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
2159         num_tc = mqprio->num_tc;
2160
2161         if (!num_tc) {
2162                 netdev_reset_tc(ndev);
2163                 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2164
2165                 /* Reset all ring priorities to 0 */
2166                 for (i = 0; i < priv->num_tx_rings; i++) {
2167                         tx_ring = priv->tx_ring[i];
2168                         enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0);
2169                 }
2170
2171                 return 0;
2172         }
2173
2174         /* Check if we have enough BD rings available to accommodate all TCs */
2175         if (num_tc > num_stack_tx_queues) {
2176                 netdev_err(ndev, "Max %d traffic classes supported\n",
2177                            priv->num_tx_rings);
2178                 return -EINVAL;
2179         }
2180
2181         /* For the moment, we use only one BD ring per TC.
2182          *
2183          * Configure num_tc BD rings with increasing priorities.
2184          */
2185         for (i = 0; i < num_tc; i++) {
2186                 tx_ring = priv->tx_ring[i];
2187                 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i);
2188         }
2189
2190         /* Reset the number of netdev queues based on the TC count */
2191         netif_set_real_num_tx_queues(ndev, num_tc);
2192
2193         netdev_set_num_tc(ndev, num_tc);
2194
2195         /* Each TC is associated with one netdev queue */
2196         for (i = 0; i < num_tc; i++)
2197                 netdev_set_tc_queue(ndev, i, 1, i);
2198
2199         return 0;
2200 }
2201
2202 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
2203                    void *type_data)
2204 {
2205         switch (type) {
2206         case TC_SETUP_QDISC_MQPRIO:
2207                 return enetc_setup_tc_mqprio(ndev, type_data);
2208         case TC_SETUP_QDISC_TAPRIO:
2209                 return enetc_setup_tc_taprio(ndev, type_data);
2210         case TC_SETUP_QDISC_CBS:
2211                 return enetc_setup_tc_cbs(ndev, type_data);
2212         case TC_SETUP_QDISC_ETF:
2213                 return enetc_setup_tc_txtime(ndev, type_data);
2214         case TC_SETUP_BLOCK:
2215                 return enetc_setup_tc_psfp(ndev, type_data);
2216         default:
2217                 return -EOPNOTSUPP;
2218         }
2219 }
2220
2221 static int enetc_setup_xdp_prog(struct net_device *dev, struct bpf_prog *prog,
2222                                 struct netlink_ext_ack *extack)
2223 {
2224         struct enetc_ndev_priv *priv = netdev_priv(dev);
2225         struct bpf_prog *old_prog;
2226         bool is_up;
2227         int i;
2228
2229         /* The buffer layout is changing, so we need to drain the old
2230          * RX buffers and seed new ones.
2231          */
2232         is_up = netif_running(dev);
2233         if (is_up)
2234                 dev_close(dev);
2235
2236         old_prog = xchg(&priv->xdp_prog, prog);
2237         if (old_prog)
2238                 bpf_prog_put(old_prog);
2239
2240         for (i = 0; i < priv->num_rx_rings; i++) {
2241                 struct enetc_bdr *rx_ring = priv->rx_ring[i];
2242
2243                 rx_ring->xdp.prog = prog;
2244
2245                 if (prog)
2246                         rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2247                 else
2248                         rx_ring->buffer_offset = ENETC_RXB_PAD;
2249         }
2250
2251         if (is_up)
2252                 return dev_open(dev, extack);
2253
2254         return 0;
2255 }
2256
2257 int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp)
2258 {
2259         switch (xdp->command) {
2260         case XDP_SETUP_PROG:
2261                 return enetc_setup_xdp_prog(dev, xdp->prog, xdp->extack);
2262         default:
2263                 return -EINVAL;
2264         }
2265
2266         return 0;
2267 }
2268
2269 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2270 {
2271         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2272         struct net_device_stats *stats = &ndev->stats;
2273         unsigned long packets = 0, bytes = 0;
2274         int i;
2275
2276         for (i = 0; i < priv->num_rx_rings; i++) {
2277                 packets += priv->rx_ring[i]->stats.packets;
2278                 bytes   += priv->rx_ring[i]->stats.bytes;
2279         }
2280
2281         stats->rx_packets = packets;
2282         stats->rx_bytes = bytes;
2283         bytes = 0;
2284         packets = 0;
2285
2286         for (i = 0; i < priv->num_tx_rings; i++) {
2287                 packets += priv->tx_ring[i]->stats.packets;
2288                 bytes   += priv->tx_ring[i]->stats.bytes;
2289         }
2290
2291         stats->tx_packets = packets;
2292         stats->tx_bytes = bytes;
2293
2294         return stats;
2295 }
2296
2297 static int enetc_set_rss(struct net_device *ndev, int en)
2298 {
2299         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2300         struct enetc_hw *hw = &priv->si->hw;
2301         u32 reg;
2302
2303         enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2304
2305         reg = enetc_rd(hw, ENETC_SIMR);
2306         reg &= ~ENETC_SIMR_RSSE;
2307         reg |= (en) ? ENETC_SIMR_RSSE : 0;
2308         enetc_wr(hw, ENETC_SIMR, reg);
2309
2310         return 0;
2311 }
2312
2313 static int enetc_set_psfp(struct net_device *ndev, int en)
2314 {
2315         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2316         int err;
2317
2318         if (en) {
2319                 err = enetc_psfp_enable(priv);
2320                 if (err)
2321                         return err;
2322
2323                 priv->active_offloads |= ENETC_F_QCI;
2324                 return 0;
2325         }
2326
2327         err = enetc_psfp_disable(priv);
2328         if (err)
2329                 return err;
2330
2331         priv->active_offloads &= ~ENETC_F_QCI;
2332
2333         return 0;
2334 }
2335
2336 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
2337 {
2338         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2339         int i;
2340
2341         for (i = 0; i < priv->num_rx_rings; i++)
2342                 enetc_bdr_enable_rxvlan(&priv->si->hw, i, en);
2343 }
2344
2345 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
2346 {
2347         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2348         int i;
2349
2350         for (i = 0; i < priv->num_tx_rings; i++)
2351                 enetc_bdr_enable_txvlan(&priv->si->hw, i, en);
2352 }
2353
2354 int enetc_set_features(struct net_device *ndev,
2355                        netdev_features_t features)
2356 {
2357         netdev_features_t changed = ndev->features ^ features;
2358         int err = 0;
2359
2360         if (changed & NETIF_F_RXHASH)
2361                 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2362
2363         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2364                 enetc_enable_rxvlan(ndev,
2365                                     !!(features & NETIF_F_HW_VLAN_CTAG_RX));
2366
2367         if (changed & NETIF_F_HW_VLAN_CTAG_TX)
2368                 enetc_enable_txvlan(ndev,
2369                                     !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2370
2371         if (changed & NETIF_F_HW_TC)
2372                 err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC));
2373
2374         return err;
2375 }
2376
2377 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2378 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2379 {
2380         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2381         struct hwtstamp_config config;
2382         int ao;
2383
2384         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2385                 return -EFAULT;
2386
2387         switch (config.tx_type) {
2388         case HWTSTAMP_TX_OFF:
2389                 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2390                 break;
2391         case HWTSTAMP_TX_ON:
2392                 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2393                 priv->active_offloads |= ENETC_F_TX_TSTAMP;
2394                 break;
2395         case HWTSTAMP_TX_ONESTEP_SYNC:
2396                 priv->active_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2397                 priv->active_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
2398                 break;
2399         default:
2400                 return -ERANGE;
2401         }
2402
2403         ao = priv->active_offloads;
2404         switch (config.rx_filter) {
2405         case HWTSTAMP_FILTER_NONE:
2406                 priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
2407                 break;
2408         default:
2409                 priv->active_offloads |= ENETC_F_RX_TSTAMP;
2410                 config.rx_filter = HWTSTAMP_FILTER_ALL;
2411         }
2412
2413         if (netif_running(ndev) && ao != priv->active_offloads) {
2414                 enetc_close(ndev);
2415                 enetc_open(ndev);
2416         }
2417
2418         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2419                -EFAULT : 0;
2420 }
2421
2422 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2423 {
2424         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2425         struct hwtstamp_config config;
2426
2427         config.flags = 0;
2428
2429         if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
2430                 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
2431         else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2432                 config.tx_type = HWTSTAMP_TX_ON;
2433         else
2434                 config.tx_type = HWTSTAMP_TX_OFF;
2435
2436         config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2437                             HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2438
2439         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2440                -EFAULT : 0;
2441 }
2442 #endif
2443
2444 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2445 {
2446         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2447 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2448         if (cmd == SIOCSHWTSTAMP)
2449                 return enetc_hwtstamp_set(ndev, rq);
2450         if (cmd == SIOCGHWTSTAMP)
2451                 return enetc_hwtstamp_get(ndev, rq);
2452 #endif
2453
2454         if (!priv->phylink)
2455                 return -EOPNOTSUPP;
2456
2457         return phylink_mii_ioctl(priv->phylink, rq, cmd);
2458 }
2459
2460 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
2461 {
2462         struct pci_dev *pdev = priv->si->pdev;
2463         int first_xdp_tx_ring;
2464         int i, n, err, nvec;
2465         int v_tx_rings;
2466
2467         nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
2468         /* allocate MSIX for both messaging and Rx/Tx interrupts */
2469         n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
2470
2471         if (n < 0)
2472                 return n;
2473
2474         if (n != nvec)
2475                 return -EPERM;
2476
2477         /* # of tx rings per int vector */
2478         v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
2479
2480         for (i = 0; i < priv->bdr_int_num; i++) {
2481                 struct enetc_int_vector *v;
2482                 struct enetc_bdr *bdr;
2483                 int j;
2484
2485                 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
2486                 if (!v) {
2487                         err = -ENOMEM;
2488                         goto fail;
2489                 }
2490
2491                 priv->int_vector[i] = v;
2492
2493                 bdr = &v->rx_ring;
2494                 bdr->index = i;
2495                 bdr->ndev = priv->ndev;
2496                 bdr->dev = priv->dev;
2497                 bdr->bd_count = priv->rx_bd_count;
2498                 bdr->buffer_offset = ENETC_RXB_PAD;
2499                 priv->rx_ring[i] = bdr;
2500
2501                 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
2502                 if (err) {
2503                         kfree(v);
2504                         goto fail;
2505                 }
2506
2507                 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
2508                                                  MEM_TYPE_PAGE_SHARED, NULL);
2509                 if (err) {
2510                         xdp_rxq_info_unreg(&bdr->xdp.rxq);
2511                         kfree(v);
2512                         goto fail;
2513                 }
2514
2515                 /* init defaults for adaptive IC */
2516                 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
2517                         v->rx_ictt = 0x1;
2518                         v->rx_dim_en = true;
2519                 }
2520                 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
2521                 netif_napi_add(priv->ndev, &v->napi, enetc_poll,
2522                                NAPI_POLL_WEIGHT);
2523                 v->count_tx_rings = v_tx_rings;
2524
2525                 for (j = 0; j < v_tx_rings; j++) {
2526                         int idx;
2527
2528                         /* default tx ring mapping policy */
2529                         idx = priv->bdr_int_num * j + i;
2530                         __set_bit(idx, &v->tx_rings_map);
2531                         bdr = &v->tx_ring[j];
2532                         bdr->index = idx;
2533                         bdr->ndev = priv->ndev;
2534                         bdr->dev = priv->dev;
2535                         bdr->bd_count = priv->tx_bd_count;
2536                         priv->tx_ring[idx] = bdr;
2537                 }
2538         }
2539
2540         first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
2541         priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
2542
2543         return 0;
2544
2545 fail:
2546         while (i--) {
2547                 struct enetc_int_vector *v = priv->int_vector[i];
2548                 struct enetc_bdr *rx_ring = &v->rx_ring;
2549
2550                 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2551                 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2552                 netif_napi_del(&v->napi);
2553                 cancel_work_sync(&v->rx_dim.work);
2554                 kfree(v);
2555         }
2556
2557         pci_free_irq_vectors(pdev);
2558
2559         return err;
2560 }
2561
2562 void enetc_free_msix(struct enetc_ndev_priv *priv)
2563 {
2564         int i;
2565
2566         for (i = 0; i < priv->bdr_int_num; i++) {
2567                 struct enetc_int_vector *v = priv->int_vector[i];
2568                 struct enetc_bdr *rx_ring = &v->rx_ring;
2569
2570                 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
2571                 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
2572                 netif_napi_del(&v->napi);
2573                 cancel_work_sync(&v->rx_dim.work);
2574         }
2575
2576         for (i = 0; i < priv->num_rx_rings; i++)
2577                 priv->rx_ring[i] = NULL;
2578
2579         for (i = 0; i < priv->num_tx_rings; i++)
2580                 priv->tx_ring[i] = NULL;
2581
2582         for (i = 0; i < priv->bdr_int_num; i++) {
2583                 kfree(priv->int_vector[i]);
2584                 priv->int_vector[i] = NULL;
2585         }
2586
2587         /* disable all MSIX for this device */
2588         pci_free_irq_vectors(priv->si->pdev);
2589 }
2590
2591 static void enetc_kfree_si(struct enetc_si *si)
2592 {
2593         char *p = (char *)si - si->pad;
2594
2595         kfree(p);
2596 }
2597
2598 static void enetc_detect_errata(struct enetc_si *si)
2599 {
2600         if (si->pdev->revision == ENETC_REV1)
2601                 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
2602 }
2603
2604 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
2605 {
2606         struct enetc_si *si, *p;
2607         struct enetc_hw *hw;
2608         size_t alloc_size;
2609         int err, len;
2610
2611         pcie_flr(pdev);
2612         err = pci_enable_device_mem(pdev);
2613         if (err) {
2614                 dev_err(&pdev->dev, "device enable failed\n");
2615                 return err;
2616         }
2617
2618         /* set up for high or low dma */
2619         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2620         if (err) {
2621                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2622                 if (err) {
2623                         dev_err(&pdev->dev,
2624                                 "DMA configuration failed: 0x%x\n", err);
2625                         goto err_dma;
2626                 }
2627         }
2628
2629         err = pci_request_mem_regions(pdev, name);
2630         if (err) {
2631                 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
2632                 goto err_pci_mem_reg;
2633         }
2634
2635         pci_set_master(pdev);
2636
2637         alloc_size = sizeof(struct enetc_si);
2638         if (sizeof_priv) {
2639                 /* align priv to 32B */
2640                 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
2641                 alloc_size += sizeof_priv;
2642         }
2643         /* force 32B alignment for enetc_si */
2644         alloc_size += ENETC_SI_ALIGN - 1;
2645
2646         p = kzalloc(alloc_size, GFP_KERNEL);
2647         if (!p) {
2648                 err = -ENOMEM;
2649                 goto err_alloc_si;
2650         }
2651
2652         si = PTR_ALIGN(p, ENETC_SI_ALIGN);
2653         si->pad = (char *)si - (char *)p;
2654
2655         pci_set_drvdata(pdev, si);
2656         si->pdev = pdev;
2657         hw = &si->hw;
2658
2659         len = pci_resource_len(pdev, ENETC_BAR_REGS);
2660         hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
2661         if (!hw->reg) {
2662                 err = -ENXIO;
2663                 dev_err(&pdev->dev, "ioremap() failed\n");
2664                 goto err_ioremap;
2665         }
2666         if (len > ENETC_PORT_BASE)
2667                 hw->port = hw->reg + ENETC_PORT_BASE;
2668         if (len > ENETC_GLOBAL_BASE)
2669                 hw->global = hw->reg + ENETC_GLOBAL_BASE;
2670
2671         enetc_detect_errata(si);
2672
2673         return 0;
2674
2675 err_ioremap:
2676         enetc_kfree_si(si);
2677 err_alloc_si:
2678         pci_release_mem_regions(pdev);
2679 err_pci_mem_reg:
2680 err_dma:
2681         pci_disable_device(pdev);
2682
2683         return err;
2684 }
2685
2686 void enetc_pci_remove(struct pci_dev *pdev)
2687 {
2688         struct enetc_si *si = pci_get_drvdata(pdev);
2689         struct enetc_hw *hw = &si->hw;
2690
2691         iounmap(hw->reg);
2692         enetc_kfree_si(si);
2693         pci_release_mem_regions(pdev);
2694         pci_disable_device(pdev);
2695 }