return err;
}
+static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx)
+{
+ ctx->curr_frag_cnt = 0;
+ ctx->total_expected_size = 0;
+ ctx->expected_frag_cnt = 0;
+ ctx->skb_head = NULL;
+ ctx->skb_tail = NULL;
+ ctx->reuse_frags = false;
+}
+
static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
{
struct gve_rx_ring *rx = &priv->rx[idx];
rx->cnt = 0;
rx->db_threshold = priv->rx_desc_cnt / 2;
rx->desc.seqno = 1;
+
+ /* Allocating half-page buffers allows page-flipping which is faster
+ * than copying or allocating new pages.
+ */
+ rx->packet_buffer_size = PAGE_SIZE / 2;
+ gve_rx_ctx_clear(&rx->ctx);
gve_rx_add_to_block(priv, idx);
return 0;
return PKT_HASH_TYPE_L2;
}
+static u16 gve_rx_ctx_padding(struct gve_rx_ctx *ctx)
+{
+ return (ctx->curr_frag_cnt == 0) ? GVE_RX_PAD : 0;
+}
+
static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi,
struct gve_rx_slot_page_info *page_info,
- u16 len)
+ u16 packet_buffer_size, u16 len,
+ struct gve_rx_ctx *ctx)
{
- struct sk_buff *skb = napi_get_frags(napi);
+ u32 offset = page_info->page_offset + gve_rx_ctx_padding(ctx);
+ struct sk_buff *skb;
+
+ if (!ctx->skb_head)
+ ctx->skb_head = napi_get_frags(napi);
- if (unlikely(!skb))
+ if (unlikely(!ctx->skb_head))
return NULL;
- skb_add_rx_frag(skb, 0, page_info->page,
- page_info->page_offset +
- GVE_RX_PAD, len, PAGE_SIZE / 2);
+ skb = ctx->skb_head;
+ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page,
+ offset, len, packet_buffer_size);
return skb;
}
*(slot_addr) ^= offset;
}
-static bool gve_rx_can_flip_buffers(struct net_device *netdev)
-{
- return PAGE_SIZE >= 4096
- ? netdev->mtu + GVE_RX_PAD + ETH_HLEN <= PAGE_SIZE / 2 : false;
-}
-
static int gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info *page_info)
{
int pagecount = page_count(page_info->page);
gve_rx_raw_addressing(struct device *dev, struct net_device *netdev,
struct gve_rx_slot_page_info *page_info, u16 len,
struct napi_struct *napi,
- union gve_rx_data_slot *data_slot)
+ union gve_rx_data_slot *data_slot,
+ u16 packet_buffer_size, struct gve_rx_ctx *ctx)
{
- struct sk_buff *skb;
+ struct sk_buff *skb = gve_rx_add_frags(napi, page_info, packet_buffer_size, len, ctx);
- skb = gve_rx_add_frags(napi, page_info, len);
if (!skb)
return NULL;
u16 len, struct napi_struct *napi,
union gve_rx_data_slot *data_slot)
{
+ struct gve_rx_ctx *ctx = &rx->ctx;
struct sk_buff *skb;
/* if raw_addressing mode is not enabled gvnic can only receive into
* choice is to copy the data out of it so that we can return it to the
* device.
*/
- if (page_info->can_flip) {
- skb = gve_rx_add_frags(napi, page_info, len);
+ if (ctx->reuse_frags) {
+ skb = gve_rx_add_frags(napi, page_info, rx->packet_buffer_size, len, ctx);
/* No point in recycling if we didn't get the skb */
if (skb) {
/* Make sure that the page isn't freed. */
gve_rx_flip_buff(page_info, &data_slot->qpl_offset);
}
} else {
- skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD);
+ const u16 padding = gve_rx_ctx_padding(ctx);
+
+ skb = gve_rx_copy(netdev, napi, page_info, len, padding, ctx);
if (skb) {
u64_stats_update_begin(&rx->statss);
- rx->rx_copied_pkt++;
+ rx->rx_frag_copy_cnt++;
u64_stats_update_end(&rx->statss);
}
}
return skb;
}
-static bool gve_rx(struct gve_rx_ring *rx, struct gve_rx_desc *rx_desc,
- netdev_features_t feat, u32 idx)
+#define GVE_PKTCONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x))
+static u16 gve_rx_get_fragment_size(struct gve_rx_ctx *ctx, struct gve_rx_desc *desc)
{
+ return be16_to_cpu(desc->len) - gve_rx_ctx_padding(ctx);
+}
+
+static bool gve_rx_ctx_init(struct gve_rx_ctx *ctx, struct gve_rx_ring *rx)
+{
+ bool qpl_mode = !rx->data.raw_addressing, packet_size_error = false;
+ bool buffer_error = false, desc_error = false, seqno_error = false;
struct gve_rx_slot_page_info *page_info;
struct gve_priv *priv = rx->gve;
- struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
- struct net_device *dev = priv->dev;
- union gve_rx_data_slot *data_slot;
- struct sk_buff *skb = NULL;
- dma_addr_t page_bus;
- void *va;
- u16 len;
+ u32 idx = rx->cnt & rx->mask;
+ bool reuse_frags, can_flip;
+ struct gve_rx_desc *desc;
+ u16 packet_size = 0;
+ u16 n_frags = 0;
+ int recycle;
- /* Prefetch two packet pages ahead, we will need it soon. */
- page_info = &rx->data.page_info[(idx + 2) & rx->mask];
- va = page_info->page_address + GVE_RX_PAD +
- page_info->page_offset;
+ /** In QPL mode, we only flip buffers when all buffers containing the packet
+ * can be flipped. RDA can_flip decisions will be made later, per frag.
+ */
+ can_flip = qpl_mode;
+ reuse_frags = can_flip;
+ do {
+ u16 frag_size;
+
+ n_frags++;
+ desc = &rx->desc.desc_ring[idx];
+ desc_error = unlikely(desc->flags_seq & GVE_RXF_ERR) || desc_error;
+ if (GVE_SEQNO(desc->flags_seq) != rx->desc.seqno) {
+ seqno_error = true;
+ netdev_warn(priv->dev,
+ "RX seqno error: want=%d, got=%d, dropping packet and scheduling reset.",
+ rx->desc.seqno, GVE_SEQNO(desc->flags_seq));
+ }
+ frag_size = be16_to_cpu(desc->len);
+ packet_size += frag_size;
+ if (frag_size > rx->packet_buffer_size) {
+ packet_size_error = true;
+ netdev_warn(priv->dev,
+ "RX fragment error: packet_buffer_size=%d, frag_size=%d, droping packet.",
+ rx->packet_buffer_size, be16_to_cpu(desc->len));
+ }
+ page_info = &rx->data.page_info[idx];
+ if (can_flip) {
+ recycle = gve_rx_can_recycle_buffer(page_info);
+ reuse_frags = reuse_frags && recycle > 0;
+ buffer_error = buffer_error || unlikely(recycle < 0);
+ }
+ idx = (idx + 1) & rx->mask;
+ rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
+ } while (GVE_PKTCONT_BIT_IS_SET(desc->flags_seq));
- prefetch(page_info->page); /* Kernel page struct. */
- prefetch(va); /* Packet header. */
- prefetch(va + 64); /* Next cacheline too. */
+ prefetch(rx->desc.desc_ring + idx);
- /* drop this packet */
- if (unlikely(rx_desc->flags_seq & GVE_RXF_ERR)) {
+ ctx->curr_frag_cnt = 0;
+ ctx->total_expected_size = packet_size - GVE_RX_PAD;
+ ctx->expected_frag_cnt = n_frags;
+ ctx->skb_head = NULL;
+ ctx->reuse_frags = reuse_frags;
+
+ if (ctx->expected_frag_cnt > 1) {
u64_stats_update_begin(&rx->statss);
- rx->rx_desc_err_dropped_pkt++;
+ rx->rx_cont_packet_cnt++;
+ u64_stats_update_end(&rx->statss);
+ }
+ if (ctx->total_expected_size > priv->rx_copybreak && !ctx->reuse_frags && qpl_mode) {
+ u64_stats_update_begin(&rx->statss);
+ rx->rx_copied_pkt++;
u64_stats_update_end(&rx->statss);
+ }
+
+ if (unlikely(buffer_error || seqno_error || packet_size_error)) {
+ gve_schedule_reset(priv);
return false;
}
- len = be16_to_cpu(rx_desc->len) - GVE_RX_PAD;
- page_info = &rx->data.page_info[idx];
+ if (unlikely(desc_error)) {
+ u64_stats_update_begin(&rx->statss);
+ rx->rx_desc_err_dropped_pkt++;
+ u64_stats_update_end(&rx->statss);
+ return false;
+ }
+ return true;
+}
- data_slot = &rx->data.data_ring[idx];
- page_bus = (rx->data.raw_addressing) ?
- be64_to_cpu(data_slot->addr) & GVE_DATA_SLOT_ADDR_PAGE_MASK :
- rx->data.qpl->page_buses[idx];
- dma_sync_single_for_cpu(&priv->pdev->dev, page_bus,
- PAGE_SIZE, DMA_FROM_DEVICE);
+static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
+ struct gve_rx_slot_page_info *page_info, struct napi_struct *napi,
+ u16 len, union gve_rx_data_slot *data_slot)
+{
+ struct net_device *netdev = priv->dev;
+ struct gve_rx_ctx *ctx = &rx->ctx;
+ struct sk_buff *skb = NULL;
- if (len <= priv->rx_copybreak) {
+ if (len <= priv->rx_copybreak && ctx->expected_frag_cnt == 1) {
/* Just copy small packets */
- skb = gve_rx_copy(dev, napi, page_info, len, GVE_RX_PAD);
- u64_stats_update_begin(&rx->statss);
- rx->rx_copied_pkt++;
- rx->rx_copybreak_pkt++;
- u64_stats_update_end(&rx->statss);
+ skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD, ctx);
+ if (skb) {
+ u64_stats_update_begin(&rx->statss);
+ rx->rx_copied_pkt++;
+ rx->rx_frag_copy_cnt++;
+ rx->rx_copybreak_pkt++;
+ } u64_stats_update_end(&rx->statss);
} else {
- u8 can_flip = gve_rx_can_flip_buffers(dev);
- int recycle = 0;
+ if (rx->data.raw_addressing) {
+ int recycle = gve_rx_can_recycle_buffer(page_info);
- if (can_flip) {
- recycle = gve_rx_can_recycle_buffer(page_info);
- if (recycle < 0) {
- if (!rx->data.raw_addressing)
- gve_schedule_reset(priv);
- return false;
+ if (unlikely(recycle < 0)) {
+ gve_schedule_reset(priv);
+ return NULL;
}
- }
-
- page_info->can_flip = can_flip && recycle;
- if (rx->data.raw_addressing) {
- skb = gve_rx_raw_addressing(&priv->pdev->dev, dev,
+ page_info->can_flip = recycle;
+ if (page_info->can_flip) {
+ u64_stats_update_begin(&rx->statss);
+ rx->rx_frag_flip_cnt++;
+ u64_stats_update_end(&rx->statss);
+ }
+ skb = gve_rx_raw_addressing(&priv->pdev->dev, netdev,
page_info, len, napi,
- data_slot);
+ data_slot,
+ rx->packet_buffer_size, ctx);
} else {
- skb = gve_rx_qpl(&priv->pdev->dev, dev, rx,
+ if (ctx->reuse_frags) {
+ u64_stats_update_begin(&rx->statss);
+ rx->rx_frag_flip_cnt++;
+ u64_stats_update_end(&rx->statss);
+ }
+ skb = gve_rx_qpl(&priv->pdev->dev, netdev, rx,
page_info, len, napi, data_slot);
}
}
+ return skb;
+}
- if (!skb) {
- u64_stats_update_begin(&rx->statss);
- rx->rx_skb_alloc_fail++;
- u64_stats_update_end(&rx->statss);
- return false;
+static bool gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
+ u64 *packet_size_bytes, u32 *work_done)
+{
+ struct gve_rx_slot_page_info *page_info;
+ struct gve_rx_ctx *ctx = &rx->ctx;
+ union gve_rx_data_slot *data_slot;
+ struct gve_priv *priv = rx->gve;
+ struct gve_rx_desc *first_desc;
+ struct sk_buff *skb = NULL;
+ struct gve_rx_desc *desc;
+ struct napi_struct *napi;
+ dma_addr_t page_bus;
+ u32 work_cnt = 0;
+ void *va;
+ u32 idx;
+ u16 len;
+
+ idx = rx->cnt & rx->mask;
+ first_desc = &rx->desc.desc_ring[idx];
+ desc = first_desc;
+ napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
+
+ if (unlikely(!gve_rx_ctx_init(ctx, rx)))
+ goto skb_alloc_fail;
+
+ while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
+ /* Prefetch two packet buffers ahead, we will need it soon. */
+ page_info = &rx->data.page_info[(idx + 2) & rx->mask];
+ va = page_info->page_address + page_info->page_offset;
+
+ prefetch(page_info->page); /* Kernel page struct. */
+ prefetch(va); /* Packet header. */
+ prefetch(va + 64); /* Next cacheline too. */
+
+ len = gve_rx_get_fragment_size(ctx, desc);
+
+ page_info = &rx->data.page_info[idx];
+ data_slot = &rx->data.data_ring[idx];
+ page_bus = rx->data.raw_addressing ?
+ be64_to_cpu(data_slot->addr) - page_info->page_offset :
+ rx->data.qpl->page_buses[idx];
+ dma_sync_single_for_cpu(&priv->pdev->dev, page_bus, PAGE_SIZE, DMA_FROM_DEVICE);
+
+ skb = gve_rx_skb(priv, rx, page_info, napi, len, data_slot);
+ if (!skb) {
+ u64_stats_update_begin(&rx->statss);
+ rx->rx_skb_alloc_fail++;
+ u64_stats_update_end(&rx->statss);
+ goto skb_alloc_fail;
+ }
+
+ ctx->curr_frag_cnt++;
+ rx->cnt++;
+ idx = rx->cnt & rx->mask;
+ work_cnt++;
+ desc = &rx->desc.desc_ring[idx];
}
if (likely(feat & NETIF_F_RXCSUM)) {
/* NIC passes up the partial sum */
- if (rx_desc->csum)
+ if (first_desc->csum)
skb->ip_summed = CHECKSUM_COMPLETE;
else
skb->ip_summed = CHECKSUM_NONE;
- skb->csum = csum_unfold(rx_desc->csum);
+ skb->csum = csum_unfold(first_desc->csum);
}
/* parse flags & pass relevant info up */
if (likely(feat & NETIF_F_RXHASH) &&
- gve_needs_rss(rx_desc->flags_seq))
- skb_set_hash(skb, be32_to_cpu(rx_desc->rss_hash),
- gve_rss_type(rx_desc->flags_seq));
+ gve_needs_rss(first_desc->flags_seq))
+ skb_set_hash(skb, be32_to_cpu(first_desc->rss_hash),
+ gve_rss_type(first_desc->flags_seq));
+ *packet_size_bytes = skb->len + (skb->protocol ? ETH_HLEN : 0);
+ *work_done = work_cnt;
if (skb_is_nonlinear(skb))
napi_gro_frags(napi);
else
napi_gro_receive(napi, skb);
+
+ gve_rx_ctx_clear(ctx);
return true;
+
+skb_alloc_fail:
+ if (napi->skb)
+ napi_free_frags(napi);
+ *packet_size_bytes = 0;
+ *work_done = ctx->expected_frag_cnt;
+ while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
+ rx->cnt++;
+ ctx->curr_frag_cnt++;
+ }
+ gve_rx_ctx_clear(ctx);
+ return false;
}
bool gve_rx_work_pending(struct gve_rx_ring *rx)
union gve_rx_data_slot *data_slot =
&rx->data.data_ring[idx];
struct device *dev = &priv->pdev->dev;
-
gve_rx_free_buffer(dev, page_info, data_slot);
page_info->page = NULL;
if (gve_rx_alloc_buffer(priv, dev, page_info,
static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
netdev_features_t feat)
{
+ u32 work_done = 0, total_packet_cnt = 0, ok_packet_cnt = 0;
struct gve_priv *priv = rx->gve;
- u32 work_done = 0, packets = 0;
+ u32 idx = rx->cnt & rx->mask;
struct gve_rx_desc *desc;
- u32 cnt = rx->cnt;
- u32 idx = cnt & rx->mask;
u64 bytes = 0;
- desc = rx->desc.desc_ring + idx;
+ desc = &rx->desc.desc_ring[idx];
while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
work_done < budget) {
+ u64 packet_size_bytes = 0;
+ u32 work_cnt = 0;
bool dropped;
netif_info(priv, rx_status, priv->dev,
rx->q_num, GVE_SEQNO(desc->flags_seq),
rx->desc.seqno);
- /* prefetch two descriptors ahead */
- prefetch(rx->desc.desc_ring + ((cnt + 2) & rx->mask));
-
- dropped = !gve_rx(rx, desc, feat, idx);
+ dropped = !gve_rx(rx, feat, &packet_size_bytes, &work_cnt);
if (!dropped) {
- bytes += be16_to_cpu(desc->len) - GVE_RX_PAD;
- packets++;
+ bytes += packet_size_bytes;
+ ok_packet_cnt++;
}
- cnt++;
- idx = cnt & rx->mask;
- desc = rx->desc.desc_ring + idx;
- rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
- work_done++;
+ total_packet_cnt++;
+ idx = rx->cnt & rx->mask;
+ desc = &rx->desc.desc_ring[idx];
+ work_done += work_cnt;
}
- if (!work_done && rx->fill_cnt - cnt > rx->db_threshold)
+ if (!work_done && rx->fill_cnt - rx->cnt > rx->db_threshold)
return 0;
if (work_done) {
u64_stats_update_begin(&rx->statss);
- rx->rpackets += packets;
+ rx->rpackets += ok_packet_cnt;
rx->rbytes += bytes;
u64_stats_update_end(&rx->statss);
- rx->cnt = cnt;
}
/* restock ring slots */
if (!rx->data.raw_addressing) {
/* In QPL mode buffs are refilled as the desc are processed */
rx->fill_cnt += work_done;
- } else if (rx->fill_cnt - cnt <= rx->db_threshold) {
+ } else if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
/* In raw addressing mode buffs are only refilled if the avail
* falls below a threshold.
*/
/* If we were not able to completely refill buffers, we'll want
* to schedule this queue for work again to refill buffers.
*/
- if (rx->fill_cnt - cnt <= rx->db_threshold) {
+ if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
gve_rx_write_doorbell(priv, rx);
return budget;
}
}
gve_rx_write_doorbell(priv, rx);
- return work_done;
+ return total_packet_cnt;
}
int gve_rx_poll(struct gve_notify_block *block, int budget)
projects / platform / kernel / linux-starfive.git / commitdiff