bpf: Add xdp dynptrs

Add xdp dynptrs, which are dynptrs whose underlying pointer points
to a xdp_buff. The dynptr acts on xdp data. xdp dynptrs have two main
benefits. One is that they allow operations on sizes that are not
statically known at compile-time (eg variable-sized accesses).
Another is that parsing the packet data through dynptrs (instead of
through direct access of xdp->data and xdp->data_end) can be more
ergonomic and less brittle (eg does not need manual if checking for
being within bounds of data_end).

For reads and writes on the dynptr, this includes reading/writing
from/to and across fragments. Data slices through the bpf_dynptr_data
API are not supported; instead bpf_dynptr_slice() and
bpf_dynptr_slice_rdwr() should be used.

For examples of how xdp dynptrs can be used, please see the attached
selftests.

Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Link: https://lore.kernel.org/r/20230301154953.641654-9-joannelkoong@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index e7436d7615b0d66ad4b8904c885f3c7f4bda3dd9..23ec684e660d55297ca43b53f314ccf8132af903 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -610,11 +610,15 @@ enum bpf_type_flag {
        /* DYNPTR points to sk_buff */
        DYNPTR_TYPE_SKB         = BIT(15 + BPF_BASE_TYPE_BITS),
 
+       /* DYNPTR points to xdp_buff */
+       DYNPTR_TYPE_XDP         = BIT(16 + BPF_BASE_TYPE_BITS),
+
        __BPF_TYPE_FLAG_MAX,
        __BPF_TYPE_LAST_FLAG    = __BPF_TYPE_FLAG_MAX - 1,
 };
 
-#define DYNPTR_TYPE_FLAG_MASK  (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB)
+#define DYNPTR_TYPE_FLAG_MASK  (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
+                                | DYNPTR_TYPE_XDP)
 
 /* Max number of base types. */
 #define BPF_BASE_TYPE_LIMIT    (1UL << BPF_BASE_TYPE_BITS)
@@ -1151,6 +1155,8 @@ enum bpf_dynptr_type {
        BPF_DYNPTR_TYPE_RINGBUF,
        /* Underlying data is a sk_buff */
        BPF_DYNPTR_TYPE_SKB,
+       /* Underlying data is a xdp_buff */
+       BPF_DYNPTR_TYPE_XDP,
 };
 
 int bpf_dynptr_check_size(u32 size);

diff --git a/include/linux/filter.h b/include/linux/filter.h
index de18e844d15ed211c1a55ec8f9006421e3713179..3f6992261ec5901e9da15cfc56e8ad4cebd6baa9 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1546,6 +1546,8 @@ static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u64 index
 int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len);
 int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
                          u32 len, u64 flags);
+int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len);
+int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len);
 #else /* CONFIG_NET */
 static inline int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset,
                                       void *to, u32 len)
@@ -1558,6 +1560,18 @@ static inline int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset,
 {
        return -EOPNOTSUPP;
 }
+
+static inline int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset,
+                                      void *buf, u32 len)
+{
+       return -EOPNOTSUPP;
+}
+
+static inline int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset,
+                                       void *buf, u32 len)
+{
+       return -EOPNOTSUPP;
+}
 #endif /* CONFIG_NET */
 
 #endif /* __LINUX_FILTER_H__ */

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index d0351d30e5515466f519e1846f77ca4497e3edcd..faa304c926cf6ee53f773d84893f3d36a6948ee1 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -5344,7 +5344,7 @@ union bpf_attr {
  *             *len* must be a statically known value. The returned data slice
  *             is invalidated whenever the dynptr is invalidated.
  *
- *             skb type dynptrs may not use bpf_dynptr_data. They should
+ *             skb and xdp type dynptrs may not use bpf_dynptr_data. They should
  *             instead use bpf_dynptr_slice and bpf_dynptr_slice_rdwr.
  *     Return
  *             Pointer to the underlying dynptr data, NULL if the dynptr is

diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index e8e2414d1587cffe5e5aca80e5007985b9fe0fce..114a875a05b11bf9fd43119ac9c2fef90feeca72 100644 (file)
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -1530,6 +1530,8 @@ BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern
                return 0;
        case BPF_DYNPTR_TYPE_SKB:
                return __bpf_skb_load_bytes(src->data, src->offset + offset, dst, len);
+       case BPF_DYNPTR_TYPE_XDP:
+               return __bpf_xdp_load_bytes(src->data, src->offset + offset, dst, len);
        default:
                WARN_ONCE(true, "bpf_dynptr_read: unknown dynptr type %d\n", type);
                return -EFAULT;
@@ -1576,6 +1578,10 @@ BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, v
        case BPF_DYNPTR_TYPE_SKB:
                return __bpf_skb_store_bytes(dst->data, dst->offset + offset, src, len,
                                             flags);
+       case BPF_DYNPTR_TYPE_XDP:
+               if (flags)
+                       return -EINVAL;
+               return __bpf_xdp_store_bytes(dst->data, dst->offset + offset, src, len);
        default:
                WARN_ONCE(true, "bpf_dynptr_write: unknown dynptr type %d\n", type);
                return -EFAULT;
@@ -1615,7 +1621,8 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
        case BPF_DYNPTR_TYPE_RINGBUF:
                return (unsigned long)(ptr->data + ptr->offset + offset);
        case BPF_DYNPTR_TYPE_SKB:
-               /* skb dynptrs should use bpf_dynptr_slice / bpf_dynptr_slice_rdwr */
+       case BPF_DYNPTR_TYPE_XDP:
+               /* skb and xdp dynptrs should use bpf_dynptr_slice / bpf_dynptr_slice_rdwr */
                return 0;
        default:
                WARN_ONCE(true, "bpf_dynptr_data: unknown dynptr type %d\n", type);

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 4f5fce16543baee7a11d0d243fe15c1538eb93ce..5e42946e53abcb0fd85dfc1cc85ba2568907303b 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -752,6 +752,8 @@ static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
                return BPF_DYNPTR_TYPE_RINGBUF;
        case DYNPTR_TYPE_SKB:
                return BPF_DYNPTR_TYPE_SKB;
+       case DYNPTR_TYPE_XDP:
+               return BPF_DYNPTR_TYPE_XDP;
        default:
                return BPF_DYNPTR_TYPE_INVALID;
        }
@@ -6300,6 +6302,9 @@ static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn
                        case DYNPTR_TYPE_SKB:
                                err_extra = "skb ";
                                break;
+                       case DYNPTR_TYPE_XDP:
+                               err_extra = "xdp ";
+                               break;
                        default:
                                err_extra = "<unknown>";
                                break;
@@ -8943,6 +8948,7 @@ enum special_kfunc_type {
        KF_bpf_rbtree_add,
        KF_bpf_rbtree_first,
        KF_bpf_dynptr_from_skb,
+       KF_bpf_dynptr_from_xdp,
 };
 
 BTF_SET_START(special_kfunc_set)
@@ -8958,6 +8964,7 @@ BTF_ID(func, bpf_rbtree_remove)
 BTF_ID(func, bpf_rbtree_add)
 BTF_ID(func, bpf_rbtree_first)
 BTF_ID(func, bpf_dynptr_from_skb)
+BTF_ID(func, bpf_dynptr_from_xdp)
 BTF_SET_END(special_kfunc_set)
 
 BTF_ID_LIST(special_kfunc_list)
@@ -8975,6 +8982,7 @@ BTF_ID(func, bpf_rbtree_remove)
 BTF_ID(func, bpf_rbtree_add)
 BTF_ID(func, bpf_rbtree_first)
 BTF_ID(func, bpf_dynptr_from_skb)
+BTF_ID(func, bpf_dynptr_from_xdp)
 
 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta)
 {
@@ -9731,6 +9739,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 
                        if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
                                dynptr_arg_type |= DYNPTR_TYPE_SKB;
+                       else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp])
+                               dynptr_arg_type |= DYNPTR_TYPE_XDP;
 
                        ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type);
                        if (ret < 0)

diff --git a/net/core/filter.c b/net/core/filter.c
index f3afa31a9b10c1768c1d52205d533bcdc103dbc4..c692046fa7f677f96a9730033142bda0443e75ce 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -3839,7 +3839,7 @@ static const struct bpf_func_proto sk_skb_change_head_proto = {
        .arg3_type      = ARG_ANYTHING,
 };
 
-BPF_CALL_1(bpf_xdp_get_buff_len, struct  xdp_buff*, xdp)
+BPF_CALL_1(bpf_xdp_get_buff_len, struct xdp_buff*, xdp)
 {
        return xdp_get_buff_len(xdp);
 }
@@ -3999,6 +3999,11 @@ static const struct bpf_func_proto bpf_xdp_load_bytes_proto = {
        .arg4_type      = ARG_CONST_SIZE,
 };
 
+int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
+{
+       return ____bpf_xdp_load_bytes(xdp, offset, buf, len);
+}
+
 BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset,
           void *, buf, u32, len)
 {
@@ -4026,6 +4031,11 @@ static const struct bpf_func_proto bpf_xdp_store_bytes_proto = {
        .arg4_type      = ARG_CONST_SIZE,
 };
 
+int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
+{
+       return ____bpf_xdp_store_bytes(xdp, offset, buf, len);
+}
+
 static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset)
 {
        struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
@@ -11648,6 +11658,19 @@ __bpf_kfunc int bpf_dynptr_from_skb(struct sk_buff *skb, u64 flags,
 
        return 0;
 }
+
+__bpf_kfunc int bpf_dynptr_from_xdp(struct xdp_buff *xdp, u64 flags,
+                                   struct bpf_dynptr_kern *ptr__uninit)
+{
+       if (flags) {
+               bpf_dynptr_set_null(ptr__uninit);
+               return -EINVAL;
+       }
+
+       bpf_dynptr_init(ptr__uninit, xdp, BPF_DYNPTR_TYPE_XDP, 0, xdp_get_buff_len(xdp));
+
+       return 0;
+}
 __diag_pop();
 
 int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags,
@@ -11668,11 +11691,20 @@ BTF_SET8_START(bpf_kfunc_check_set_skb)
 BTF_ID_FLAGS(func, bpf_dynptr_from_skb)
 BTF_SET8_END(bpf_kfunc_check_set_skb)
 
+BTF_SET8_START(bpf_kfunc_check_set_xdp)
+BTF_ID_FLAGS(func, bpf_dynptr_from_xdp)
+BTF_SET8_END(bpf_kfunc_check_set_xdp)
+
 static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
        .owner = THIS_MODULE,
        .set = &bpf_kfunc_check_set_skb,
 };
 
+static const struct btf_kfunc_id_set bpf_kfunc_set_xdp = {
+       .owner = THIS_MODULE,
+       .set = &bpf_kfunc_check_set_xdp,
+};
+
 static int __init bpf_kfunc_init(void)
 {
        int ret;
@@ -11685,6 +11717,7 @@ static int __init bpf_kfunc_init(void)
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_OUT, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_IN, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_XMIT, &bpf_kfunc_set_skb);
-       return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
+       ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
+       return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &bpf_kfunc_set_xdp);
 }
 late_initcall(bpf_kfunc_init);

diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index d0351d30e5515466f519e1846f77ca4497e3edcd..faa304c926cf6ee53f773d84893f3d36a6948ee1 100644 (file)
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -5344,7 +5344,7 @@ union bpf_attr {
  *             *len* must be a statically known value. The returned data slice
  *             is invalidated whenever the dynptr is invalidated.
  *
- *             skb type dynptrs may not use bpf_dynptr_data. They should
+ *             skb and xdp type dynptrs may not use bpf_dynptr_data. They should
  *             instead use bpf_dynptr_slice and bpf_dynptr_slice_rdwr.
  *     Return
  *             Pointer to the underlying dynptr data, NULL if the dynptr is