Merge tag 'kbuild-v5.15' of git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy...

[platform/kernel/linux-starfive.git] / include / linux / filter.h

diff --git a/include/linux/filter.h b/include/linux/filter.h
index c1711c9..4a93c12 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -572,7 +572,8 @@ struct bpf_prog {
                                kprobe_override:1, /* Do we override a kprobe? */
                                has_callchain_buf:1, /* callchain buffer allocated? */
                                enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
-                               call_get_stack:1; /* Do we call bpf_get_stack() or bpf_get_stackid() */
+                               call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
+                               call_get_func_ip:1; /* Do we call get_func_ip() */
        enum bpf_prog_type      type;           /* Type of BPF program */
        enum bpf_attach_type    expected_attach_type; /* For some prog types */
        u32                     len;            /* Number of filter blocks */
@@ -597,25 +598,38 @@ struct sk_filter {
 
 DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
 
-#define __BPF_PROG_RUN(prog, ctx, dfunc)       ({                      \
-       u32 __ret;                                                      \
-       cant_migrate();                                                 \
-       if (static_branch_unlikely(&bpf_stats_enabled_key)) {           \
-               struct bpf_prog_stats *__stats;                         \
-               u64 __start = sched_clock();                            \
-               __ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func);   \
-               __stats = this_cpu_ptr(prog->stats);                    \
-               u64_stats_update_begin(&__stats->syncp);                \
-               __stats->cnt++;                                         \
-               __stats->nsecs += sched_clock() - __start;              \
-               u64_stats_update_end(&__stats->syncp);                  \
-       } else {                                                        \
-               __ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func);   \
-       }                                                               \
-       __ret; })
-
-#define BPF_PROG_RUN(prog, ctx)                                                \
-       __BPF_PROG_RUN(prog, ctx, bpf_dispatcher_nop_func)
+typedef unsigned int (*bpf_dispatcher_fn)(const void *ctx,
+                                         const struct bpf_insn *insnsi,
+                                         unsigned int (*bpf_func)(const void *,
+                                                                  const struct bpf_insn *));
+
+static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog,
+                                         const void *ctx,
+                                         bpf_dispatcher_fn dfunc)
+{
+       u32 ret;
+
+       cant_migrate();
+       if (static_branch_unlikely(&bpf_stats_enabled_key)) {
+               struct bpf_prog_stats *stats;
+               u64 start = sched_clock();
+
+               ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
+               stats = this_cpu_ptr(prog->stats);
+               u64_stats_update_begin(&stats->syncp);
+               stats->cnt++;
+               stats->nsecs += sched_clock() - start;
+               u64_stats_update_end(&stats->syncp);
+       } else {
+               ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
+       }
+       return ret;
+}
+
+static __always_inline u32 bpf_prog_run(const struct bpf_prog *prog, const void *ctx)
+{
+       return __bpf_prog_run(prog, ctx, bpf_dispatcher_nop_func);
+}
 
 /*
  * Use in preemptible and therefore migratable context to make sure that
@@ -634,7 +648,7 @@ static inline u32 bpf_prog_run_pin_on_cpu(const struct bpf_prog *prog,
        u32 ret;
 
        migrate_disable();
-       ret = __BPF_PROG_RUN(prog, ctx, bpf_dispatcher_nop_func);
+       ret = bpf_prog_run(prog, ctx);
        migrate_enable();
        return ret;
 }
@@ -707,7 +721,7 @@ static inline void bpf_restore_data_end(
        cb->data_end = saved_data_end;
 }
 
-static inline u8 *bpf_skb_cb(struct sk_buff *skb)
+static inline u8 *bpf_skb_cb(const struct sk_buff *skb)
 {
        /* eBPF programs may read/write skb->cb[] area to transfer meta
         * data between tail calls. Since this also needs to work with
@@ -728,8 +742,9 @@ static inline u8 *bpf_skb_cb(struct sk_buff *skb)
 
 /* Must be invoked with migration disabled */
 static inline u32 __bpf_prog_run_save_cb(const struct bpf_prog *prog,
-                                        struct sk_buff *skb)
+                                        const void *ctx)
 {
+       const struct sk_buff *skb = ctx;
        u8 *cb_data = bpf_skb_cb(skb);
        u8 cb_saved[BPF_SKB_CB_LEN];
        u32 res;
@@ -739,7 +754,7 @@ static inline u32 __bpf_prog_run_save_cb(const struct bpf_prog *prog,
                memset(cb_data, 0, sizeof(cb_saved));
        }
 
-       res = BPF_PROG_RUN(prog, skb);
+       res = bpf_prog_run(prog, skb);
 
        if (unlikely(prog->cb_access))
                memcpy(cb_data, cb_saved, sizeof(cb_saved));
@@ -773,6 +788,10 @@ static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog,
 
 DECLARE_BPF_DISPATCHER(xdp)
 
+DECLARE_STATIC_KEY_FALSE(bpf_master_redirect_enabled_key);
+
+u32 xdp_master_redirect(struct xdp_buff *xdp);
+
 static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
                                            struct xdp_buff *xdp)
 {
@@ -780,7 +799,14 @@ static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
         * under local_bh_disable(), which provides the needed RCU protection
         * for accessing map entries.
         */
-       return __BPF_PROG_RUN(prog, xdp, BPF_DISPATCHER_FUNC(xdp));
+       u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp));
+
+       if (static_branch_unlikely(&bpf_master_redirect_enabled_key)) {
+               if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev))
+                       act = xdp_master_redirect(xdp);
+       }
+
+       return act;
 }
 
 void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog);
@@ -1426,7 +1452,7 @@ static inline bool bpf_sk_lookup_run_v4(struct net *net, int protocol,
                };
                u32 act;
 
-               act = BPF_PROG_SK_LOOKUP_RUN_ARRAY(run_array, ctx, BPF_PROG_RUN);
+               act = BPF_PROG_SK_LOOKUP_RUN_ARRAY(run_array, ctx, bpf_prog_run);
                if (act == SK_PASS) {
                        selected_sk = ctx.selected_sk;
                        no_reuseport = ctx.no_reuseport;
@@ -1464,7 +1490,7 @@ static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol,
                };
                u32 act;
 
-               act = BPF_PROG_SK_LOOKUP_RUN_ARRAY(run_array, ctx, BPF_PROG_RUN);
+               act = BPF_PROG_SK_LOOKUP_RUN_ARRAY(run_array, ctx, bpf_prog_run);
                if (act == SK_PASS) {
                        selected_sk = ctx.selected_sk;
                        no_reuseport = ctx.no_reuseport;