drivers/net/ethernet/aquantia/atlantic/aq_ring.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Atlantic Network Driver
   3  *
   4  * Copyright (C) 2014-2019 aQuantia Corporation
   5  * Copyright (C) 2019-2020 Marvell International Ltd.
   6  */
   7
   8 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */
   9
  10 #include "aq_nic.h"
  11 #include "aq_hw.h"
  12 #include "aq_hw_utils.h"
  13 #include "aq_ptp.h"
  14 #include "aq_vec.h"
  15 #include "aq_main.h"
  16
  17 #include <net/xdp.h>
  18 #include <linux/filter.h>
  19 #include <linux/bpf_trace.h>
  20 #include <linux/netdevice.h>
  21 #include <linux/etherdevice.h>
  22
  23 static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff,
  24                                struct xdp_buff *xdp)
  25 {
  26         struct skb_shared_info *sinfo;
  27         int i;
  28
  29         if (xdp_buff_has_frags(xdp)) {
  30                 sinfo = xdp_get_shared_info_from_buff(xdp);
  31
  32                 for (i = 0; i < sinfo->nr_frags; i++) {
  33                         skb_frag_t *frag = &sinfo->frags[i];
  34
  35                         page_ref_inc(skb_frag_page(frag));
  36                 }
  37         }
  38         page_ref_inc(buff->rxdata.page);
  39 }
  40
  41 static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev)
  42 {
  43         unsigned int len = PAGE_SIZE << rxpage->order;
  44
  45         dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE);
  46
  47         /* Drop the ref for being in the ring. */
  48         __free_pages(rxpage->page, rxpage->order);
  49         rxpage->page = NULL;
  50 }
  51
  52 static int aq_alloc_rxpages(struct aq_rxpage *rxpage, struct aq_ring_s *rx_ring)
  53 {
  54         struct device *dev = aq_nic_get_dev(rx_ring->aq_nic);
  55         unsigned int order = rx_ring->page_order;
  56         struct page *page;
  57         int ret = -ENOMEM;
  58         dma_addr_t daddr;
  59
  60         page = dev_alloc_pages(order);
  61         if (unlikely(!page))
  62                 goto err_exit;
  63
  64         daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order,
  65                              DMA_FROM_DEVICE);
  66
  67         if (unlikely(dma_mapping_error(dev, daddr)))
  68                 goto free_page;
  69
  70         rxpage->page = page;
  71         rxpage->daddr = daddr;
  72         rxpage->order = order;
  73         rxpage->pg_off = rx_ring->page_offset;
  74
  75         return 0;
  76
  77 free_page:
  78         __free_pages(page, order);
  79
  80 err_exit:
  81         return ret;
  82 }
  83
  84 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf)
  85 {
  86         unsigned int order = self->page_order;
  87         u16 page_offset = self->page_offset;
  88         u16 frame_max = self->frame_max;
  89         u16 tail_size = self->tail_size;
  90         int ret;
  91
  92         if (rxbuf->rxdata.page) {
  93                 /* One means ring is the only user and can reuse */
  94                 if (page_ref_count(rxbuf->rxdata.page) > 1) {
  95                         /* Try reuse buffer */
  96                         rxbuf->rxdata.pg_off += frame_max + page_offset +
  97                                                 tail_size;
  98                         if (rxbuf->rxdata.pg_off + frame_max + tail_size <=
  99                             (PAGE_SIZE << order)) {
 100                                 u64_stats_update_begin(&self->stats.rx.syncp);
 101                                 self->stats.rx.pg_flips++;
 102                                 u64_stats_update_end(&self->stats.rx.syncp);
 103
 104                         } else {
 105                                 /* Buffer exhausted. We have other users and
 106                                  * should release this page and realloc
 107                                  */
 108                                 aq_free_rxpage(&rxbuf->rxdata,
 109                                                aq_nic_get_dev(self->aq_nic));
 110                                 u64_stats_update_begin(&self->stats.rx.syncp);
 111                                 self->stats.rx.pg_losts++;
 112                                 u64_stats_update_end(&self->stats.rx.syncp);
 113                         }
 114                 } else {
 115                         rxbuf->rxdata.pg_off = page_offset;
 116                         u64_stats_update_begin(&self->stats.rx.syncp);
 117                         self->stats.rx.pg_reuses++;
 118                         u64_stats_update_end(&self->stats.rx.syncp);
 119                 }
 120         }
 121
 122         if (!rxbuf->rxdata.page) {
 123                 ret = aq_alloc_rxpages(&rxbuf->rxdata, self);
 124                 if (ret) {
 125                         u64_stats_update_begin(&self->stats.rx.syncp);
 126                         self->stats.rx.alloc_fails++;
 127                         u64_stats_update_end(&self->stats.rx.syncp);
 128                 }
 129                 return ret;
 130         }
 131
 132         return 0;
 133 }
 134
 135 static int aq_ring_alloc(struct aq_ring_s *self,
 136                          struct aq_nic_s *aq_nic)
 137 {
 138         int err = 0;
 139
 140         self->buff_ring =
 141                 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL);
 142
 143         if (!self->buff_ring) {
 144                 err = -ENOMEM;
 145                 goto err_exit;
 146         }
 147
 148         self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic),
 149                                            self->size * self->dx_size,
 150                                            &self->dx_ring_pa, GFP_KERNEL);
 151         if (!self->dx_ring) {
 152                 err = -ENOMEM;
 153                 goto err_exit;
 154         }
 155
 156 err_exit:
 157         if (err < 0) {
 158                 aq_ring_free(self);
 159         }
 160
 161         return err;
 162 }
 163
 164 int aq_ring_tx_alloc(struct aq_ring_s *self,
 165                      struct aq_nic_s *aq_nic,
 166                      unsigned int idx,
 167                      struct aq_nic_cfg_s *aq_nic_cfg)
 168 {
 169         self->aq_nic = aq_nic;
 170         self->idx = idx;
 171         self->size = aq_nic_cfg->txds;
 172         self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size;
 173
 174         return aq_ring_alloc(self, aq_nic);
 175 }
 176
 177 int aq_ring_rx_alloc(struct aq_ring_s *self,
 178                      struct aq_nic_s *aq_nic,
 179                      unsigned int idx,
 180                      struct aq_nic_cfg_s *aq_nic_cfg)
 181 {
 182         self->aq_nic = aq_nic;
 183         self->idx = idx;
 184         self->size = aq_nic_cfg->rxds;
 185         self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size;
 186         self->xdp_prog = aq_nic->xdp_prog;
 187         self->frame_max = AQ_CFG_RX_FRAME_MAX;
 188
 189         /* Only order-2 is allowed if XDP is enabled */
 190         if (READ_ONCE(self->xdp_prog)) {
 191                 self->page_offset = AQ_XDP_HEADROOM;
 192                 self->page_order = AQ_CFG_XDP_PAGEORDER;
 193                 self->tail_size = AQ_XDP_TAILROOM;
 194         } else {
 195                 self->page_offset = 0;
 196                 self->page_order = fls(self->frame_max / PAGE_SIZE +
 197                                        (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1;
 198                 if (aq_nic_cfg->rxpageorder > self->page_order)
 199                         self->page_order = aq_nic_cfg->rxpageorder;
 200                 self->tail_size = 0;
 201         }
 202
 203         return aq_ring_alloc(self, aq_nic);
 204 }
 205
 206 int
 207 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic,
 208                       unsigned int idx, unsigned int size, unsigned int dx_size)
 209 {
 210         struct device *dev = aq_nic_get_dev(aq_nic);
 211         size_t sz = size * dx_size + AQ_CFG_RXDS_DEF;
 212
 213         memset(self, 0, sizeof(*self));
 214
 215         self->aq_nic = aq_nic;
 216         self->idx = idx;
 217         self->size = size;
 218         self->dx_size = dx_size;
 219
 220         self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa,
 221                                            GFP_KERNEL);
 222         if (!self->dx_ring) {
 223                 aq_ring_free(self);
 224                 return -ENOMEM;
 225         }
 226
 227         return 0;
 228 }
 229
 230 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type)
 231 {
 232         self->hw_head = 0;
 233         self->sw_head = 0;
 234         self->sw_tail = 0;
 235         self->ring_type = ring_type;
 236
 237         if (self->ring_type == ATL_RING_RX)
 238                 u64_stats_init(&self->stats.rx.syncp);
 239         else
 240                 u64_stats_init(&self->stats.tx.syncp);
 241
 242         return 0;
 243 }
 244
 245 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i,
 246                                        unsigned int t)
 247 {
 248         return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t));
 249 }
 250
 251 void aq_ring_update_queue_state(struct aq_ring_s *ring)
 252 {
 253         if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX)
 254                 aq_ring_queue_stop(ring);
 255         else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES)
 256                 aq_ring_queue_wake(ring);
 257 }
 258
 259 void aq_ring_queue_wake(struct aq_ring_s *ring)
 260 {
 261         struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic);
 262
 263         if (__netif_subqueue_stopped(ndev,
 264                                      AQ_NIC_RING2QMAP(ring->aq_nic,
 265                                                       ring->idx))) {
 266                 netif_wake_subqueue(ndev,
 267                                     AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx));
 268                 u64_stats_update_begin(&ring->stats.tx.syncp);
 269                 ring->stats.tx.queue_restarts++;
 270                 u64_stats_update_end(&ring->stats.tx.syncp);
 271         }
 272 }
 273
 274 void aq_ring_queue_stop(struct aq_ring_s *ring)
 275 {
 276         struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic);
 277
 278         if (!__netif_subqueue_stopped(ndev,
 279                                       AQ_NIC_RING2QMAP(ring->aq_nic,
 280                                                        ring->idx)))
 281                 netif_stop_subqueue(ndev,
 282                                     AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx));
 283 }
 284
 285 bool aq_ring_tx_clean(struct aq_ring_s *self)
 286 {
 287         struct device *dev = aq_nic_get_dev(self->aq_nic);
 288         unsigned int budget;
 289
 290         for (budget = AQ_CFG_TX_CLEAN_BUDGET;
 291              budget && self->sw_head != self->hw_head; budget--) {
 292                 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
 293
 294                 if (likely(buff->is_mapped)) {
 295                         if (unlikely(buff->is_sop)) {
 296                                 if (!buff->is_eop &&
 297                                     buff->eop_index != 0xffffU &&
 298                                     (!aq_ring_dx_in_range(self->sw_head,
 299                                                 buff->eop_index,
 300                                                 self->hw_head)))
 301                                         break;
 302
 303                                 dma_unmap_single(dev, buff->pa, buff->len,
 304                                                  DMA_TO_DEVICE);
 305                         } else {
 306                                 dma_unmap_page(dev, buff->pa, buff->len,
 307                                                DMA_TO_DEVICE);
 308                         }
 309                 }
 310
 311                 if (likely(!buff->is_eop))
 312                         goto out;
 313
 314                 if (buff->skb) {
 315                         u64_stats_update_begin(&self->stats.tx.syncp);
 316                         ++self->stats.tx.packets;
 317                         self->stats.tx.bytes += buff->skb->len;
 318                         u64_stats_update_end(&self->stats.tx.syncp);
 319                         dev_kfree_skb_any(buff->skb);
 320                 } else if (buff->xdpf) {
 321                         u64_stats_update_begin(&self->stats.tx.syncp);
 322                         ++self->stats.tx.packets;
 323                         self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf);
 324                         u64_stats_update_end(&self->stats.tx.syncp);
 325                         xdp_return_frame_rx_napi(buff->xdpf);
 326                 }
 327
 328 out:
 329                 buff->skb = NULL;
 330                 buff->xdpf = NULL;
 331                 buff->pa = 0U;
 332                 buff->eop_index = 0xffffU;
 333                 self->sw_head = aq_ring_next_dx(self, self->sw_head);
 334         }
 335
 336         return !!budget;
 337 }
 338
 339 static void aq_rx_checksum(struct aq_ring_s *self,
 340                            struct aq_ring_buff_s *buff,
 341                            struct sk_buff *skb)
 342 {
 343         if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM))
 344                 return;
 345
 346         if (unlikely(buff->is_cso_err)) {
 347                 u64_stats_update_begin(&self->stats.rx.syncp);
 348                 ++self->stats.rx.errors;
 349                 u64_stats_update_end(&self->stats.rx.syncp);
 350                 skb->ip_summed = CHECKSUM_NONE;
 351                 return;
 352         }
 353         if (buff->is_ip_cso) {
 354                 __skb_incr_checksum_unnecessary(skb);
 355         } else {
 356                 skb->ip_summed = CHECKSUM_NONE;
 357         }
 358
 359         if (buff->is_udp_cso || buff->is_tcp_cso)
 360                 __skb_incr_checksum_unnecessary(skb);
 361 }
 362
 363 int aq_xdp_xmit(struct net_device *dev, int num_frames,
 364                 struct xdp_frame **frames, u32 flags)
 365 {
 366         struct aq_nic_s *aq_nic = netdev_priv(dev);
 367         unsigned int vec, i, drop = 0;
 368         int cpu = smp_processor_id();
 369         struct aq_nic_cfg_s *aq_cfg;
 370         struct aq_ring_s *ring;
 371
 372         aq_cfg = aq_nic_get_cfg(aq_nic);
 373         vec = cpu % aq_cfg->vecs;
 374         ring = aq_nic->aq_ring_tx[AQ_NIC_CFG_TCVEC2RING(aq_cfg, 0, vec)];
 375
 376         for (i = 0; i < num_frames; i++) {
 377                 struct xdp_frame *xdpf = frames[i];
 378
 379                 if (aq_nic_xmit_xdpf(aq_nic, ring, xdpf) == NETDEV_TX_BUSY)
 380                         drop++;
 381         }
 382
 383         return num_frames - drop;
 384 }
 385
 386 static struct sk_buff *aq_xdp_build_skb(struct xdp_buff *xdp,
 387                                         struct net_device *dev,
 388                                         struct aq_ring_buff_s *buff)
 389 {
 390         struct xdp_frame *xdpf;
 391         struct sk_buff *skb;
 392
 393         xdpf = xdp_convert_buff_to_frame(xdp);
 394         if (unlikely(!xdpf))
 395                 return NULL;
 396
 397         skb = xdp_build_skb_from_frame(xdpf, dev);
 398         if (!skb)
 399                 return NULL;
 400
 401         aq_get_rxpages_xdp(buff, xdp);
 402         return skb;
 403 }
 404
 405 static struct sk_buff *aq_xdp_run_prog(struct aq_nic_s *aq_nic,
 406                                        struct xdp_buff *xdp,
 407                                        struct aq_ring_s *rx_ring,
 408                                        struct aq_ring_buff_s *buff)
 409 {
 410         int result = NETDEV_TX_BUSY;
 411         struct aq_ring_s *tx_ring;
 412         struct xdp_frame *xdpf;
 413         struct bpf_prog *prog;
 414         u32 act = XDP_ABORTED;
 415         struct sk_buff *skb;
 416
 417         u64_stats_update_begin(&rx_ring->stats.rx.syncp);
 418         ++rx_ring->stats.rx.packets;
 419         rx_ring->stats.rx.bytes += xdp_get_buff_len(xdp);
 420         u64_stats_update_end(&rx_ring->stats.rx.syncp);
 421
 422         prog = READ_ONCE(rx_ring->xdp_prog);
 423         if (!prog)
 424                 return aq_xdp_build_skb(xdp, aq_nic->ndev, buff);
 425
 426         prefetchw(xdp->data_hard_start); /* xdp_frame write */
 427
 428         /* single buffer XDP program, but packet is multi buffer, aborted */
 429         if (xdp_buff_has_frags(xdp) && !prog->aux->xdp_has_frags)
 430                 goto out_aborted;
 431
 432         act = bpf_prog_run_xdp(prog, xdp);
 433         switch (act) {
 434         case XDP_PASS:
 435                 skb = aq_xdp_build_skb(xdp, aq_nic->ndev, buff);
 436                 if (!skb)
 437                         goto out_aborted;
 438                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
 439                 ++rx_ring->stats.rx.xdp_pass;
 440                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
 441                 return skb;
 442         case XDP_TX:
 443                 xdpf = xdp_convert_buff_to_frame(xdp);
 444                 if (unlikely(!xdpf))
 445                         goto out_aborted;
 446                 tx_ring = aq_nic->aq_ring_tx[rx_ring->idx];
 447                 result = aq_nic_xmit_xdpf(aq_nic, tx_ring, xdpf);
 448                 if (result == NETDEV_TX_BUSY)
 449                         goto out_aborted;
 450                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
 451                 ++rx_ring->stats.rx.xdp_tx;
 452                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
 453                 aq_get_rxpages_xdp(buff, xdp);
 454                 break;
 455         case XDP_REDIRECT:
 456                 if (xdp_do_redirect(aq_nic->ndev, xdp, prog) < 0)
 457                         goto out_aborted;
 458                 xdp_do_flush();
 459                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
 460                 ++rx_ring->stats.rx.xdp_redirect;
 461                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
 462                 aq_get_rxpages_xdp(buff, xdp);
 463                 break;
 464         default:
 465                 fallthrough;
 466         case XDP_ABORTED:
 467 out_aborted:
 468                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
 469                 ++rx_ring->stats.rx.xdp_aborted;
 470                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
 471                 trace_xdp_exception(aq_nic->ndev, prog, act);
 472                 bpf_warn_invalid_xdp_action(aq_nic->ndev, prog, act);
 473                 break;
 474         case XDP_DROP:
 475                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
 476                 ++rx_ring->stats.rx.xdp_drop;
 477                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
 478                 break;
 479         }
 480
 481         return ERR_PTR(-result);
 482 }
 483
 484 static bool aq_add_rx_fragment(struct device *dev,
 485                                struct aq_ring_s *ring,
 486                                struct aq_ring_buff_s *buff,
 487                                struct xdp_buff *xdp)
 488 {
 489         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
 490         struct aq_ring_buff_s *buff_ = buff;
 491
 492         memset(sinfo, 0, sizeof(*sinfo));
 493         do {
 494                 skb_frag_t *frag;
 495
 496                 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS))
 497                         return true;
 498
 499                 frag = &sinfo->frags[sinfo->nr_frags++];
 500                 buff_ = &ring->buff_ring[buff_->next];
 501                 dma_sync_single_range_for_cpu(dev,
 502                                               buff_->rxdata.daddr,
 503                                               buff_->rxdata.pg_off,
 504                                               buff_->len,
 505                                               DMA_FROM_DEVICE);
 506                 sinfo->xdp_frags_size += buff_->len;
 507                 skb_frag_fill_page_desc(frag, buff_->rxdata.page,
 508                                         buff_->rxdata.pg_off,
 509                                         buff_->len);
 510
 511                 buff_->is_cleaned = 1;
 512
 513                 buff->is_ip_cso &= buff_->is_ip_cso;
 514                 buff->is_udp_cso &= buff_->is_udp_cso;
 515                 buff->is_tcp_cso &= buff_->is_tcp_cso;
 516                 buff->is_cso_err |= buff_->is_cso_err;
 517
 518                 if (page_is_pfmemalloc(buff_->rxdata.page))
 519                         xdp_buff_set_frag_pfmemalloc(xdp);
 520
 521         } while (!buff_->is_eop);
 522
 523         xdp_buff_set_frags_flag(xdp);
 524
 525         return false;
 526 }
 527
 528 static int __aq_ring_rx_clean(struct aq_ring_s *self, struct napi_struct *napi,
 529                               int *work_done, int budget)
 530 {
 531         struct net_device *ndev = aq_nic_get_ndev(self->aq_nic);
 532         int err = 0;
 533
 534         for (; (self->sw_head != self->hw_head) && budget;
 535                 self->sw_head = aq_ring_next_dx(self, self->sw_head),
 536                 --budget, ++(*work_done)) {
 537                 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
 538                 bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self);
 539                 struct aq_ring_buff_s *buff_ = NULL;
 540                 struct sk_buff *skb = NULL;
 541                 unsigned int next_ = 0U;
 542                 unsigned int i = 0U;
 543                 u16 hdr_len;
 544
 545                 if (buff->is_cleaned)
 546                         continue;
 547
 548                 if (!buff->is_eop) {
 549                         unsigned int frag_cnt = 0U;
 550                         buff_ = buff;
 551                         do {
 552                                 bool is_rsc_completed = true;
 553
 554                                 if (buff_->next >= self->size) {
 555                                         err = -EIO;
 556                                         goto err_exit;
 557                                 }
 558
 559                                 frag_cnt++;
 560                                 next_ = buff_->next,
 561                                 buff_ = &self->buff_ring[next_];
 562                                 is_rsc_completed =
 563                                         aq_ring_dx_in_range(self->sw_head,
 564                                                             next_,
 565                                                             self->hw_head);
 566
 567                                 if (unlikely(!is_rsc_completed) ||
 568                                                 frag_cnt > MAX_SKB_FRAGS) {
 569                                         err = 0;
 570                                         goto err_exit;
 571                                 }
 572
 573                                 buff->is_error |= buff_->is_error;
 574                                 buff->is_cso_err |= buff_->is_cso_err;
 575
 576                         } while (!buff_->is_eop);
 577
 578                         if (buff->is_error ||
 579                             (buff->is_lro && buff->is_cso_err)) {
 580                                 buff_ = buff;
 581                                 do {
 582                                         if (buff_->next >= self->size) {
 583                                                 err = -EIO;
 584                                                 goto err_exit;
 585                                         }
 586                                         next_ = buff_->next,
 587                                         buff_ = &self->buff_ring[next_];
 588
 589                                         buff_->is_cleaned = true;
 590                                 } while (!buff_->is_eop);
 591
 592                                 u64_stats_update_begin(&self->stats.rx.syncp);
 593                                 ++self->stats.rx.errors;
 594                                 u64_stats_update_end(&self->stats.rx.syncp);
 595                                 continue;
 596                         }
 597                 }
 598
 599                 if (buff->is_error) {
 600                         u64_stats_update_begin(&self->stats.rx.syncp);
 601                         ++self->stats.rx.errors;
 602                         u64_stats_update_end(&self->stats.rx.syncp);
 603                         continue;
 604                 }
 605
 606                 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic),
 607                                               buff->rxdata.daddr,
 608                                               buff->rxdata.pg_off,
 609                                               buff->len, DMA_FROM_DEVICE);
 610
 611                 skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE);
 612                 if (unlikely(!skb)) {
 613                         u64_stats_update_begin(&self->stats.rx.syncp);
 614                         self->stats.rx.skb_alloc_fails++;
 615                         u64_stats_update_end(&self->stats.rx.syncp);
 616                         err = -ENOMEM;
 617                         goto err_exit;
 618                 }
 619                 if (is_ptp_ring)
 620                         buff->len -=
 621                                 aq_ptp_extract_ts(self->aq_nic, skb_hwtstamps(skb),
 622                                                   aq_buf_vaddr(&buff->rxdata),
 623                                                   buff->len);
 624
 625                 hdr_len = buff->len;
 626                 if (hdr_len > AQ_CFG_RX_HDR_SIZE)
 627                         hdr_len = eth_get_headlen(skb->dev,
 628                                                   aq_buf_vaddr(&buff->rxdata),
 629                                                   AQ_CFG_RX_HDR_SIZE);
 630
 631                 memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata),
 632                        ALIGN(hdr_len, sizeof(long)));
 633
 634                 if (buff->len - hdr_len > 0) {
 635                         skb_add_rx_frag(skb, i++, buff->rxdata.page,
 636                                         buff->rxdata.pg_off + hdr_len,
 637                                         buff->len - hdr_len,
 638                                         self->frame_max);
 639                         page_ref_inc(buff->rxdata.page);
 640                 }
 641
 642                 if (!buff->is_eop) {
 643                         buff_ = buff;
 644                         do {
 645                                 next_ = buff_->next;
 646                                 buff_ = &self->buff_ring[next_];
 647
 648                                 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic),
 649                                                               buff_->rxdata.daddr,
 650                                                               buff_->rxdata.pg_off,
 651                                                               buff_->len,
 652                                                               DMA_FROM_DEVICE);
 653                                 skb_add_rx_frag(skb, i++,
 654                                                 buff_->rxdata.page,
 655                                                 buff_->rxdata.pg_off,
 656                                                 buff_->len,
 657                                                 self->frame_max);
 658                                 page_ref_inc(buff_->rxdata.page);
 659                                 buff_->is_cleaned = 1;
 660
 661                                 buff->is_ip_cso &= buff_->is_ip_cso;
 662                                 buff->is_udp_cso &= buff_->is_udp_cso;
 663                                 buff->is_tcp_cso &= buff_->is_tcp_cso;
 664                                 buff->is_cso_err |= buff_->is_cso_err;
 665
 666                         } while (!buff_->is_eop);
 667                 }
 668
 669                 if (buff->is_vlan)
 670                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 671                                                buff->vlan_rx_tag);
 672
 673                 skb->protocol = eth_type_trans(skb, ndev);
 674
 675                 aq_rx_checksum(self, buff, skb);
 676
 677                 skb_set_hash(skb, buff->rss_hash,
 678                              buff->is_hash_l4 ? PKT_HASH_TYPE_L4 :
 679                              PKT_HASH_TYPE_NONE);
 680                 /* Send all PTP traffic to 0 queue */
 681                 skb_record_rx_queue(skb,
 682                                     is_ptp_ring ? 0
 683                                                 : AQ_NIC_RING2QMAP(self->aq_nic,
 684                                                                    self->idx));
 685
 686                 u64_stats_update_begin(&self->stats.rx.syncp);
 687                 ++self->stats.rx.packets;
 688                 self->stats.rx.bytes += skb->len;
 689                 u64_stats_update_end(&self->stats.rx.syncp);
 690
 691                 napi_gro_receive(napi, skb);
 692         }
 693
 694 err_exit:
 695         return err;
 696 }
 697
 698 static int __aq_ring_xdp_clean(struct aq_ring_s *rx_ring,
 699                                struct napi_struct *napi, int *work_done,
 700                                int budget)
 701 {
 702         int frame_sz = rx_ring->page_offset + rx_ring->frame_max +
 703                        rx_ring->tail_size;
 704         struct aq_nic_s *aq_nic = rx_ring->aq_nic;
 705         bool is_rsc_completed = true;
 706         struct device *dev;
 707         int err = 0;
 708
 709         dev = aq_nic_get_dev(aq_nic);
 710         for (; (rx_ring->sw_head != rx_ring->hw_head) && budget;
 711                 rx_ring->sw_head = aq_ring_next_dx(rx_ring, rx_ring->sw_head),
 712                 --budget, ++(*work_done)) {
 713                 struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head];
 714                 bool is_ptp_ring = aq_ptp_ring(rx_ring->aq_nic, rx_ring);
 715                 struct aq_ring_buff_s *buff_ = NULL;
 716                 u16 ptp_hwtstamp_len = 0;
 717                 struct skb_shared_hwtstamps shhwtstamps;
 718                 struct sk_buff *skb = NULL;
 719                 unsigned int next_ = 0U;
 720                 struct xdp_buff xdp;
 721                 void *hard_start;
 722
 723                 if (buff->is_cleaned)
 724                         continue;
 725
 726                 if (!buff->is_eop) {
 727                         buff_ = buff;
 728                         do {
 729                                 if (buff_->next >= rx_ring->size) {
 730                                         err = -EIO;
 731                                         goto err_exit;
 732                                 }
 733                                 next_ = buff_->next;
 734                                 buff_ = &rx_ring->buff_ring[next_];
 735                                 is_rsc_completed =
 736                                         aq_ring_dx_in_range(rx_ring->sw_head,
 737                                                             next_,
 738                                                             rx_ring->hw_head);
 739
 740                                 if (unlikely(!is_rsc_completed))
 741                                         break;
 742
 743                                 buff->is_error |= buff_->is_error;
 744                                 buff->is_cso_err |= buff_->is_cso_err;
 745                         } while (!buff_->is_eop);
 746
 747                         if (!is_rsc_completed) {
 748                                 err = 0;
 749                                 goto err_exit;
 750                         }
 751                         if (buff->is_error ||
 752                             (buff->is_lro && buff->is_cso_err)) {
 753                                 buff_ = buff;
 754                                 do {
 755                                         if (buff_->next >= rx_ring->size) {
 756                                                 err = -EIO;
 757                                                 goto err_exit;
 758                                         }
 759                                         next_ = buff_->next;
 760                                         buff_ = &rx_ring->buff_ring[next_];
 761
 762                                         buff_->is_cleaned = true;
 763                                 } while (!buff_->is_eop);
 764
 765                                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
 766                                 ++rx_ring->stats.rx.errors;
 767                                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
 768                                 continue;
 769                         }
 770                 }
 771
 772                 if (buff->is_error) {
 773                         u64_stats_update_begin(&rx_ring->stats.rx.syncp);
 774                         ++rx_ring->stats.rx.errors;
 775                         u64_stats_update_end(&rx_ring->stats.rx.syncp);
 776                         continue;
 777                 }
 778
 779                 dma_sync_single_range_for_cpu(dev,
 780                                               buff->rxdata.daddr,
 781                                               buff->rxdata.pg_off,
 782                                               buff->len, DMA_FROM_DEVICE);
 783                 hard_start = page_address(buff->rxdata.page) +
 784                              buff->rxdata.pg_off - rx_ring->page_offset;
 785
 786                 if (is_ptp_ring) {
 787                         ptp_hwtstamp_len = aq_ptp_extract_ts(rx_ring->aq_nic, &shhwtstamps,
 788                                                              aq_buf_vaddr(&buff->rxdata),
 789                                                              buff->len);
 790                         buff->len -= ptp_hwtstamp_len;
 791                 }
 792
 793                 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);
 794                 xdp_prepare_buff(&xdp, hard_start, rx_ring->page_offset,
 795                                  buff->len, false);
 796                 if (!buff->is_eop) {
 797                         if (aq_add_rx_fragment(dev, rx_ring, buff, &xdp)) {
 798                                 u64_stats_update_begin(&rx_ring->stats.rx.syncp);
 799                                 ++rx_ring->stats.rx.packets;
 800                                 rx_ring->stats.rx.bytes += xdp_get_buff_len(&xdp);
 801                                 ++rx_ring->stats.rx.xdp_aborted;
 802                                 u64_stats_update_end(&rx_ring->stats.rx.syncp);
 803                                 continue;
 804                         }
 805                 }
 806
 807                 skb = aq_xdp_run_prog(aq_nic, &xdp, rx_ring, buff);
 808                 if (IS_ERR(skb) || !skb)
 809                         continue;
 810
 811                 if (ptp_hwtstamp_len > 0)
 812                         *skb_hwtstamps(skb) = shhwtstamps;
 813
 814                 if (buff->is_vlan)
 815                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 816                                                buff->vlan_rx_tag);
 817
 818                 aq_rx_checksum(rx_ring, buff, skb);
 819
 820                 skb_set_hash(skb, buff->rss_hash,
 821                              buff->is_hash_l4 ? PKT_HASH_TYPE_L4 :
 822                              PKT_HASH_TYPE_NONE);
 823                 /* Send all PTP traffic to 0 queue */
 824                 skb_record_rx_queue(skb,
 825                                     is_ptp_ring ? 0
 826                                                 : AQ_NIC_RING2QMAP(rx_ring->aq_nic,
 827                                                                    rx_ring->idx));
 828
 829                 napi_gro_receive(napi, skb);
 830         }
 831
 832 err_exit:
 833         return err;
 834 }
 835
 836 int aq_ring_rx_clean(struct aq_ring_s *self,
 837                      struct napi_struct *napi,
 838                      int *work_done,
 839                      int budget)
 840 {
 841         if (static_branch_unlikely(&aq_xdp_locking_key))
 842                 return __aq_ring_xdp_clean(self, napi, work_done, budget);
 843         else
 844                 return __aq_ring_rx_clean(self, napi, work_done, budget);
 845 }
 846
 847 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic)
 848 {
 849 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
 850         while (self->sw_head != self->hw_head) {
 851                 u64 ns;
 852
 853                 aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw,
 854                                                 self->dx_ring +
 855                                                 (self->sw_head * self->dx_size),
 856                                                 self->dx_size, &ns);
 857                 aq_ptp_tx_hwtstamp(aq_nic, ns);
 858
 859                 self->sw_head = aq_ring_next_dx(self, self->sw_head);
 860         }
 861 #endif
 862 }
 863
 864 int aq_ring_rx_fill(struct aq_ring_s *self)
 865 {
 866         struct aq_ring_buff_s *buff = NULL;
 867         int err = 0;
 868         int i = 0;
 869
 870         if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES,
 871                                            self->size / 2))
 872                 return err;
 873
 874         for (i = aq_ring_avail_dx(self); i--;
 875                 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) {
 876                 buff = &self->buff_ring[self->sw_tail];
 877
 878                 buff->flags = 0U;
 879                 buff->len = self->frame_max;
 880
 881                 err = aq_get_rxpages(self, buff);
 882                 if (err)
 883                         goto err_exit;
 884
 885                 buff->pa = aq_buf_daddr(&buff->rxdata);
 886                 buff = NULL;
 887         }
 888
 889 err_exit:
 890         return err;
 891 }
 892
 893 void aq_ring_rx_deinit(struct aq_ring_s *self)
 894 {
 895         if (!self)
 896                 return;
 897
 898         for (; self->sw_head != self->sw_tail;
 899                 self->sw_head = aq_ring_next_dx(self, self->sw_head)) {
 900                 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
 901
 902                 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic));
 903         }
 904 }
 905
 906 void aq_ring_free(struct aq_ring_s *self)
 907 {
 908         if (!self)
 909                 return;
 910
 911         kfree(self->buff_ring);
 912         self->buff_ring = NULL;
 913
 914         if (self->dx_ring) {
 915                 dma_free_coherent(aq_nic_get_dev(self->aq_nic),
 916                                   self->size * self->dx_size, self->dx_ring,
 917                                   self->dx_ring_pa);
 918                 self->dx_ring = NULL;
 919         }
 920 }
 921
 922 void aq_ring_hwts_rx_free(struct aq_ring_s *self)
 923 {
 924         if (!self)
 925                 return;
 926
 927         if (self->dx_ring) {
 928                 dma_free_coherent(aq_nic_get_dev(self->aq_nic),
 929                                   self->size * self->dx_size + AQ_CFG_RXDS_DEF,
 930                                   self->dx_ring, self->dx_ring_pa);
 931                 self->dx_ring = NULL;
 932         }
 933 }
 934
 935 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data)
 936 {
 937         unsigned int count;
 938         unsigned int start;
 939
 940         if (self->ring_type == ATL_RING_RX) {
 941                 /* This data should mimic aq_ethtool_queue_rx_stat_names structure */
 942                 do {
 943                         count = 0;
 944                         start = u64_stats_fetch_begin(&self->stats.rx.syncp);
 945                         data[count] = self->stats.rx.packets;
 946                         data[++count] = self->stats.rx.jumbo_packets;
 947                         data[++count] = self->stats.rx.lro_packets;
 948                         data[++count] = self->stats.rx.errors;
 949                         data[++count] = self->stats.rx.alloc_fails;
 950                         data[++count] = self->stats.rx.skb_alloc_fails;
 951                         data[++count] = self->stats.rx.polls;
 952                         data[++count] = self->stats.rx.pg_flips;
 953                         data[++count] = self->stats.rx.pg_reuses;
 954                         data[++count] = self->stats.rx.pg_losts;
 955                         data[++count] = self->stats.rx.xdp_aborted;
 956                         data[++count] = self->stats.rx.xdp_drop;
 957                         data[++count] = self->stats.rx.xdp_pass;
 958                         data[++count] = self->stats.rx.xdp_tx;
 959                         data[++count] = self->stats.rx.xdp_invalid;
 960                         data[++count] = self->stats.rx.xdp_redirect;
 961                 } while (u64_stats_fetch_retry(&self->stats.rx.syncp, start));
 962         } else {
 963                 /* This data should mimic aq_ethtool_queue_tx_stat_names structure */
 964                 do {
 965                         count = 0;
 966                         start = u64_stats_fetch_begin(&self->stats.tx.syncp);
 967                         data[count] = self->stats.tx.packets;
 968                         data[++count] = self->stats.tx.queue_restarts;
 969                 } while (u64_stats_fetch_retry(&self->stats.tx.syncp, start));
 970         }
 971
 972         return ++count;
 973 }