drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2016-2017 Broadcom Limited
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  */
   9 #include <linux/kernel.h>
  10 #include <linux/errno.h>
  11 #include <linux/pci.h>
  12 #include <linux/netdevice.h>
  13 #include <linux/etherdevice.h>
  14 #include <linux/if_vlan.h>
  15 #include <linux/bpf.h>
  16 #include <linux/bpf_trace.h>
  17 #include <linux/filter.h>
  18 #include <net/page_pool.h>
  19 #include "bnxt_hsi.h"
  20 #include "bnxt.h"
  21 #include "bnxt_xdp.h"
  22
  23 DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);
  24
  25 struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
  26                                    struct bnxt_tx_ring_info *txr,
  27                                    dma_addr_t mapping, u32 len,
  28                                    struct xdp_buff *xdp)
  29 {
  30         struct skb_shared_info *sinfo;
  31         struct bnxt_sw_tx_bd *tx_buf;
  32         struct tx_bd *txbd;
  33         int num_frags = 0;
  34         u32 flags;
  35         u16 prod;
  36         int i;
  37
  38         if (xdp && xdp_buff_has_frags(xdp)) {
  39                 sinfo = xdp_get_shared_info_from_buff(xdp);
  40                 num_frags = sinfo->nr_frags;
  41         }
  42
  43         /* fill up the first buffer */
  44         prod = txr->tx_prod;
  45         tx_buf = &txr->tx_buf_ring[prod];
  46         tx_buf->nr_frags = num_frags;
  47         if (xdp)
  48                 tx_buf->page = virt_to_head_page(xdp->data);
  49
  50         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
  51         flags = (len << TX_BD_LEN_SHIFT) |
  52                 ((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT) |
  53                 bnxt_lhint_arr[len >> 9];
  54         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
  55         txbd->tx_bd_opaque = prod;
  56         txbd->tx_bd_haddr = cpu_to_le64(mapping);
  57
  58         /* now let us fill up the frags into the next buffers */
  59         for (i = 0; i < num_frags ; i++) {
  60                 skb_frag_t *frag = &sinfo->frags[i];
  61                 struct bnxt_sw_tx_bd *frag_tx_buf;
  62                 struct pci_dev *pdev = bp->pdev;
  63                 dma_addr_t frag_mapping;
  64                 int frag_len;
  65
  66                 prod = NEXT_TX(prod);
  67                 WRITE_ONCE(txr->tx_prod, prod);
  68
  69                 /* first fill up the first buffer */
  70                 frag_tx_buf = &txr->tx_buf_ring[prod];
  71                 frag_tx_buf->page = skb_frag_page(frag);
  72
  73                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
  74
  75                 frag_len = skb_frag_size(frag);
  76                 frag_mapping = skb_frag_dma_map(&pdev->dev, frag, 0,
  77                                                 frag_len, DMA_TO_DEVICE);
  78
  79                 if (unlikely(dma_mapping_error(&pdev->dev, frag_mapping)))
  80                         return NULL;
  81
  82                 dma_unmap_addr_set(frag_tx_buf, mapping, frag_mapping);
  83
  84                 flags = frag_len << TX_BD_LEN_SHIFT;
  85                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
  86                 txbd->tx_bd_haddr = cpu_to_le64(frag_mapping);
  87
  88                 len = frag_len;
  89         }
  90
  91         flags &= ~TX_BD_LEN;
  92         txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags |
  93                         TX_BD_FLAGS_PACKET_END);
  94         /* Sync TX BD */
  95         wmb();
  96         prod = NEXT_TX(prod);
  97         WRITE_ONCE(txr->tx_prod, prod);
  98
  99         return tx_buf;
 100 }
 101
 102 static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 103                             dma_addr_t mapping, u32 len, u16 rx_prod,
 104                             struct xdp_buff *xdp)
 105 {
 106         struct bnxt_sw_tx_bd *tx_buf;
 107
 108         tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp);
 109         tx_buf->rx_prod = rx_prod;
 110         tx_buf->action = XDP_TX;
 111
 112 }
 113
 114 static void __bnxt_xmit_xdp_redirect(struct bnxt *bp,
 115                                      struct bnxt_tx_ring_info *txr,
 116                                      dma_addr_t mapping, u32 len,
 117                                      struct xdp_frame *xdpf)
 118 {
 119         struct bnxt_sw_tx_bd *tx_buf;
 120
 121         tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL);
 122         tx_buf->action = XDP_REDIRECT;
 123         tx_buf->xdpf = xdpf;
 124         dma_unmap_addr_set(tx_buf, mapping, mapping);
 125         dma_unmap_len_set(tx_buf, len, 0);
 126 }
 127
 128 void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
 129 {
 130         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 131         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 132         bool rx_doorbell_needed = false;
 133         int nr_pkts = bnapi->tx_pkts;
 134         struct bnxt_sw_tx_bd *tx_buf;
 135         u16 tx_cons = txr->tx_cons;
 136         u16 last_tx_cons = tx_cons;
 137         int i, j, frags;
 138
 139         if (!budget)
 140                 return;
 141
 142         for (i = 0; i < nr_pkts; i++) {
 143                 tx_buf = &txr->tx_buf_ring[tx_cons];
 144
 145                 if (tx_buf->action == XDP_REDIRECT) {
 146                         struct pci_dev *pdev = bp->pdev;
 147
 148                         dma_unmap_single(&pdev->dev,
 149                                          dma_unmap_addr(tx_buf, mapping),
 150                                          dma_unmap_len(tx_buf, len),
 151                                          DMA_TO_DEVICE);
 152                         xdp_return_frame(tx_buf->xdpf);
 153                         tx_buf->action = 0;
 154                         tx_buf->xdpf = NULL;
 155                 } else if (tx_buf->action == XDP_TX) {
 156                         rx_doorbell_needed = true;
 157                         last_tx_cons = tx_cons;
 158
 159                         frags = tx_buf->nr_frags;
 160                         for (j = 0; j < frags; j++) {
 161                                 tx_cons = NEXT_TX(tx_cons);
 162                                 tx_buf = &txr->tx_buf_ring[tx_cons];
 163                                 page_pool_recycle_direct(rxr->page_pool, tx_buf->page);
 164                         }
 165                 }
 166                 tx_cons = NEXT_TX(tx_cons);
 167         }
 168
 169         bnapi->tx_pkts = 0;
 170         WRITE_ONCE(txr->tx_cons, tx_cons);
 171         if (rx_doorbell_needed) {
 172                 tx_buf = &txr->tx_buf_ring[last_tx_cons];
 173                 bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod);
 174
 175         }
 176 }
 177
 178 bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
 179 {
 180         struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
 181
 182         return !!xdp_prog;
 183 }
 184
 185 void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 186                         u16 cons, u8 *data_ptr, unsigned int len,
 187                         struct xdp_buff *xdp)
 188 {
 189         u32 buflen = BNXT_RX_PAGE_SIZE;
 190         struct bnxt_sw_rx_bd *rx_buf;
 191         struct pci_dev *pdev;
 192         dma_addr_t mapping;
 193         u32 offset;
 194
 195         pdev = bp->pdev;
 196         rx_buf = &rxr->rx_buf_ring[cons];
 197         offset = bp->rx_offset;
 198
 199         mapping = rx_buf->mapping - bp->rx_dma_offset;
 200         dma_sync_single_for_cpu(&pdev->dev, mapping + offset, len, bp->rx_dir);
 201
 202         xdp_init_buff(xdp, buflen, &rxr->xdp_rxq);
 203         xdp_prepare_buff(xdp, data_ptr - offset, offset, len, false);
 204 }
 205
 206 void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr,
 207                               struct xdp_buff *xdp)
 208 {
 209         struct skb_shared_info *shinfo;
 210         int i;
 211
 212         if (!xdp || !xdp_buff_has_frags(xdp))
 213                 return;
 214         shinfo = xdp_get_shared_info_from_buff(xdp);
 215         for (i = 0; i < shinfo->nr_frags; i++) {
 216                 struct page *page = skb_frag_page(&shinfo->frags[i]);
 217
 218                 page_pool_recycle_direct(rxr->page_pool, page);
 219         }
 220         shinfo->nr_frags = 0;
 221 }
 222
 223 /* returns the following:
 224  * true    - packet consumed by XDP and new buffer is allocated.
 225  * false   - packet should be passed to the stack.
 226  */
 227 bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
 228                  struct xdp_buff xdp, struct page *page, u8 **data_ptr,
 229                  unsigned int *len, u8 *event)
 230 {
 231         struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
 232         struct bnxt_tx_ring_info *txr;
 233         struct bnxt_sw_rx_bd *rx_buf;
 234         struct pci_dev *pdev;
 235         dma_addr_t mapping;
 236         u32 tx_needed = 1;
 237         void *orig_data;
 238         u32 tx_avail;
 239         u32 offset;
 240         u32 act;
 241
 242         if (!xdp_prog)
 243                 return false;
 244
 245         pdev = bp->pdev;
 246         offset = bp->rx_offset;
 247
 248         txr = rxr->bnapi->tx_ring;
 249         /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */
 250         orig_data = xdp.data;
 251
 252         act = bpf_prog_run_xdp(xdp_prog, &xdp);
 253
 254         tx_avail = bnxt_tx_avail(bp, txr);
 255         /* If the tx ring is not full, we must not update the rx producer yet
 256          * because we may still be transmitting on some BDs.
 257          */
 258         if (tx_avail != bp->tx_ring_size)
 259                 *event &= ~BNXT_RX_EVENT;
 260
 261         *len = xdp.data_end - xdp.data;
 262         if (orig_data != xdp.data) {
 263                 offset = xdp.data - xdp.data_hard_start;
 264                 *data_ptr = xdp.data_hard_start + offset;
 265         }
 266
 267         switch (act) {
 268         case XDP_PASS:
 269                 return false;
 270
 271         case XDP_TX:
 272                 rx_buf = &rxr->rx_buf_ring[cons];
 273                 mapping = rx_buf->mapping - bp->rx_dma_offset;
 274                 *event = 0;
 275
 276                 if (unlikely(xdp_buff_has_frags(&xdp))) {
 277                         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(&xdp);
 278
 279                         tx_needed += sinfo->nr_frags;
 280                         *event = BNXT_AGG_EVENT;
 281                 }
 282
 283                 if (tx_avail < tx_needed) {
 284                         trace_xdp_exception(bp->dev, xdp_prog, act);
 285                         bnxt_xdp_buff_frags_free(rxr, &xdp);
 286                         bnxt_reuse_rx_data(rxr, cons, page);
 287                         return true;
 288                 }
 289
 290                 dma_sync_single_for_device(&pdev->dev, mapping + offset, *len,
 291                                            bp->rx_dir);
 292
 293                 *event |= BNXT_TX_EVENT;
 294                 __bnxt_xmit_xdp(bp, txr, mapping + offset, *len,
 295                                 NEXT_RX(rxr->rx_prod), &xdp);
 296                 bnxt_reuse_rx_data(rxr, cons, page);
 297                 return true;
 298         case XDP_REDIRECT:
 299                 /* if we are calling this here then we know that the
 300                  * redirect is coming from a frame received by the
 301                  * bnxt_en driver.
 302                  */
 303                 rx_buf = &rxr->rx_buf_ring[cons];
 304                 mapping = rx_buf->mapping - bp->rx_dma_offset;
 305                 dma_unmap_page_attrs(&pdev->dev, mapping,
 306                                      BNXT_RX_PAGE_SIZE, bp->rx_dir,
 307                                      DMA_ATTR_WEAK_ORDERING);
 308
 309                 /* if we are unable to allocate a new buffer, abort and reuse */
 310                 if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) {
 311                         trace_xdp_exception(bp->dev, xdp_prog, act);
 312                         bnxt_xdp_buff_frags_free(rxr, &xdp);
 313                         bnxt_reuse_rx_data(rxr, cons, page);
 314                         return true;
 315                 }
 316
 317                 if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) {
 318                         trace_xdp_exception(bp->dev, xdp_prog, act);
 319                         page_pool_recycle_direct(rxr->page_pool, page);
 320                         return true;
 321                 }
 322
 323                 *event |= BNXT_REDIRECT_EVENT;
 324                 break;
 325         default:
 326                 bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act);
 327                 fallthrough;
 328         case XDP_ABORTED:
 329                 trace_xdp_exception(bp->dev, xdp_prog, act);
 330                 fallthrough;
 331         case XDP_DROP:
 332                 bnxt_xdp_buff_frags_free(rxr, &xdp);
 333                 bnxt_reuse_rx_data(rxr, cons, page);
 334                 break;
 335         }
 336         return true;
 337 }
 338
 339 int bnxt_xdp_xmit(struct net_device *dev, int num_frames,
 340                   struct xdp_frame **frames, u32 flags)
 341 {
 342         struct bnxt *bp = netdev_priv(dev);
 343         struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog);
 344         struct pci_dev *pdev = bp->pdev;
 345         struct bnxt_tx_ring_info *txr;
 346         dma_addr_t mapping;
 347         int nxmit = 0;
 348         int ring;
 349         int i;
 350
 351         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
 352             !bp->tx_nr_rings_xdp ||
 353             !xdp_prog)
 354                 return -EINVAL;
 355
 356         ring = smp_processor_id() % bp->tx_nr_rings_xdp;
 357         txr = &bp->tx_ring[ring];
 358
 359         if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING)
 360                 return -EINVAL;
 361
 362         if (static_branch_unlikely(&bnxt_xdp_locking_key))
 363                 spin_lock(&txr->xdp_tx_lock);
 364
 365         for (i = 0; i < num_frames; i++) {
 366                 struct xdp_frame *xdp = frames[i];
 367
 368                 if (!bnxt_tx_avail(bp, txr))
 369                         break;
 370
 371                 mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,
 372                                          DMA_TO_DEVICE);
 373
 374                 if (dma_mapping_error(&pdev->dev, mapping))
 375                         break;
 376
 377                 __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp);
 378                 nxmit++;
 379         }
 380
 381         if (flags & XDP_XMIT_FLUSH) {
 382                 /* Sync BD data before updating doorbell */
 383                 wmb();
 384                 bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);
 385         }
 386
 387         if (static_branch_unlikely(&bnxt_xdp_locking_key))
 388                 spin_unlock(&txr->xdp_tx_lock);
 389
 390         return nxmit;
 391 }
 392
 393 /* Under rtnl_lock */
 394 static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)
 395 {
 396         struct net_device *dev = bp->dev;
 397         int tx_xdp = 0, rc, tc;
 398         struct bpf_prog *old;
 399
 400         if (prog && !prog->aux->xdp_has_frags &&
 401             bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
 402                 netdev_warn(dev, "MTU %d larger than %d without XDP frag support.\n",
 403                             bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU);
 404                 return -EOPNOTSUPP;
 405         }
 406         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) {
 407                 netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n");
 408                 return -EOPNOTSUPP;
 409         }
 410         if (prog)
 411                 tx_xdp = bp->rx_nr_rings;
 412
 413         tc = netdev_get_num_tc(dev);
 414         if (!tc)
 415                 tc = 1;
 416         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
 417                               true, tc, tx_xdp);
 418         if (rc) {
 419                 netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n");
 420                 return rc;
 421         }
 422         if (netif_running(dev))
 423                 bnxt_close_nic(bp, true, false);
 424
 425         old = xchg(&bp->xdp_prog, prog);
 426         if (old)
 427                 bpf_prog_put(old);
 428
 429         if (prog) {
 430                 bnxt_set_rx_skb_mode(bp, true);
 431                 xdp_features_set_redirect_target(dev, true);
 432         } else {
 433                 int rx, tx;
 434
 435                 xdp_features_clear_redirect_target(dev);
 436                 bnxt_set_rx_skb_mode(bp, false);
 437                 bnxt_get_max_rings(bp, &rx, &tx, true);
 438                 if (rx > 1) {
 439                         bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS;
 440                         bp->dev->hw_features |= NETIF_F_LRO;
 441                 }
 442         }
 443         bp->tx_nr_rings_xdp = tx_xdp;
 444         bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp;
 445         bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
 446         bnxt_set_tpa_flags(bp);
 447         bnxt_set_ring_params(bp);
 448
 449         if (netif_running(dev))
 450                 return bnxt_open_nic(bp, true, false);
 451
 452         return 0;
 453 }
 454
 455 int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 456 {
 457         struct bnxt *bp = netdev_priv(dev);
 458         int rc;
 459
 460         switch (xdp->command) {
 461         case XDP_SETUP_PROG:
 462                 rc = bnxt_xdp_set(bp, xdp->prog);
 463                 break;
 464         default:
 465                 rc = -EINVAL;
 466                 break;
 467         }
 468         return rc;
 469 }
 470
 471 struct sk_buff *
 472 bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags,
 473                    struct page_pool *pool, struct xdp_buff *xdp,
 474                    struct rx_cmp_ext *rxcmp1)
 475 {
 476         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
 477
 478         if (!skb)
 479                 return NULL;
 480         skb_checksum_none_assert(skb);
 481         if (RX_CMP_L4_CS_OK(rxcmp1)) {
 482                 if (bp->dev->features & NETIF_F_RXCSUM) {
 483                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 484                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
 485                 }
 486         }
 487         xdp_update_skb_shared_info(skb, num_frags,
 488                                    sinfo->xdp_frags_size,
 489                                    BNXT_RX_PAGE_SIZE * sinfo->nr_frags,
 490                                    xdp_buff_is_frag_pfmemalloc(xdp));
 491         return skb;
 492 }