#include <linux/ptr_ring.h>
#include <linux/bpf_trace.h>
#include <linux/net_tstamp.h>
+#include <net/page_pool.h>
#define DRV_NAME "veth"
#define DRV_VERSION "1.0"
bool rx_notify_masked;
struct ptr_ring xdp_ring;
struct xdp_rxq_info xdp_rxq;
+ struct page_pool *page_pool;
};
struct veth_priv {
goto drop;
/* Allocate skb head */
- page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
+ page = page_pool_dev_alloc_pages(rq->page_pool);
if (!page)
goto drop;
nskb = build_skb(page_address(page), PAGE_SIZE);
if (!nskb) {
- put_page(page);
+ page_pool_put_full_page(rq->page_pool, page, true);
goto drop;
}
skb_reserve(nskb, VETH_XDP_HEADROOM);
+ skb_copy_header(nskb, skb);
+ skb_mark_for_recycle(nskb);
+
size = min_t(u32, skb->len, max_head_size);
if (skb_copy_bits(skb, 0, nskb->data, size)) {
consume_skb(nskb);
}
skb_put(nskb, size);
- skb_copy_header(nskb, skb);
head_off = skb_headroom(nskb) - skb_headroom(skb);
skb_headers_offset_update(nskb, head_off);
len = skb->len - off;
for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {
- page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
+ page = page_pool_dev_alloc_pages(rq->page_pool);
if (!page) {
consume_skb(nskb);
goto drop;
return done;
}
+static int veth_create_page_pool(struct veth_rq *rq)
+{
+ struct page_pool_params pp_params = {
+ .order = 0,
+ .pool_size = VETH_RING_SIZE,
+ .nid = NUMA_NO_NODE,
+ .dev = &rq->dev->dev,
+ };
+
+ rq->page_pool = page_pool_create(&pp_params);
+ if (IS_ERR(rq->page_pool)) {
+ int err = PTR_ERR(rq->page_pool);
+
+ rq->page_pool = NULL;
+ return err;
+ }
+
+ return 0;
+}
+
static int __veth_napi_enable_range(struct net_device *dev, int start, int end)
{
struct veth_priv *priv = netdev_priv(dev);
int err, i;
for (i = start; i < end; i++) {
+ err = veth_create_page_pool(&priv->rq[i]);
+ if (err)
+ goto err_page_pool;
+ }
+
+ for (i = start; i < end; i++) {
struct veth_rq *rq = &priv->rq[i];
err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL);
err_xdp_ring:
for (i--; i >= start; i--)
ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free);
+err_page_pool:
+ for (i = start; i < end; i++) {
+ page_pool_destroy(priv->rq[i].page_pool);
+ priv->rq[i].page_pool = NULL;
+ }
return err;
}
rq->rx_notify_masked = false;
ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free);
}
+
+ for (i = start; i < end; i++) {
+ page_pool_destroy(priv->rq[i].page_pool);
+ priv->rq[i].page_pool = NULL;
+ }
}
static void veth_napi_del(struct net_device *dev)