net/sched: avoid indirect classify functions on retpoline kernels

Expose the necessary tc classifier functions and wire up cls_api to use
direct calls in retpoline kernels.

Signed-off-by: Pedro Tammela <pctammela@mojatatu.com>
Reviewed-by: Jamal Hadi Salim <jhs@mojatatu.com>
Reviewed-by: Victor Nogueira <victor@mojatatu.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index 23d1cfa4f58ce90802e633e0d88fa2de1db15a33..668130f0890343a44b36d8d9b4ee6e175d1b98f4 100644 (file)
--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c
@@ -40,6 +40,7 @@
 #include <net/tc_act/tc_mpls.h>
 #include <net/tc_act/tc_gate.h>
 #include <net/flow_offload.h>
+#include <net/tc_wrapper.h>
 
 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
 
@@ -1564,7 +1565,7 @@ reclassify:
                    tp->protocol != htons(ETH_P_ALL))
                        continue;
 
-               err = tp->classify(skb, tp, res);
+               err = tc_classify(skb, tp, res);
 #ifdef CONFIG_NET_CLS_ACT
                if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
                        first_tp = orig_tp;

diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c
index d229ce99e55487f396501a5d56ac98e8d58c99d9..1b92c33b5f819365d941e5361881044af6872523 100644 (file)
--- a/net/sched/cls_basic.c
+++ b/net/sched/cls_basic.c
@@ -18,6 +18,7 @@
 #include <net/netlink.h>
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
+#include <net/tc_wrapper.h>
 
 struct basic_head {
        struct list_head        flist;
@@ -36,8 +37,9 @@ struct basic_filter {
        struct rcu_work         rwork;
 };
 
-static int basic_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                         struct tcf_result *res)
+TC_INDIRECT_SCOPE int basic_classify(struct sk_buff *skb,
+                                    const struct tcf_proto *tp,
+                                    struct tcf_result *res)
 {
        int r;
        struct basic_head *head = rcu_dereference_bh(tp->root);

diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c
index bc317b3eac124b2df292bdb61f7a2dc39ef11d51..466c26df853a052aa7ca2733dd1afe70a06909d4 100644 (file)
--- a/net/sched/cls_bpf.c
+++ b/net/sched/cls_bpf.c
@@ -19,6 +19,7 @@
 #include <net/rtnetlink.h>
 #include <net/pkt_cls.h>
 #include <net/sock.h>
+#include <net/tc_wrapper.h>
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
@@ -77,8 +78,9 @@ static int cls_bpf_exec_opcode(int code)
        }
 }
 
-static int cls_bpf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                           struct tcf_result *res)
+TC_INDIRECT_SCOPE int cls_bpf_classify(struct sk_buff *skb,
+                                      const struct tcf_proto *tp,
+                                      struct tcf_result *res)
 {
        struct cls_bpf_head *head = rcu_dereference_bh(tp->root);
        bool at_ingress = skb_at_tc_ingress(skb);

diff --git a/net/sched/cls_cgroup.c b/net/sched/cls_cgroup.c
index ed00001b528a6c63e7122fb38aa9ad9fb928acb7..bd9322d71910bfa414e96d15f9bb7cb1f29c2d97 100644 (file)
--- a/net/sched/cls_cgroup.c
+++ b/net/sched/cls_cgroup.c
@@ -13,6 +13,7 @@
 #include <net/pkt_cls.h>
 #include <net/sock.h>
 #include <net/cls_cgroup.h>
+#include <net/tc_wrapper.h>
 
 struct cls_cgroup_head {
        u32                     handle;
@@ -22,8 +23,9 @@ struct cls_cgroup_head {
        struct rcu_work         rwork;
 };
 
-static int cls_cgroup_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                              struct tcf_result *res)
+TC_INDIRECT_SCOPE int cls_cgroup_classify(struct sk_buff *skb,
+                                         const struct tcf_proto *tp,
+                                         struct tcf_result *res)
 {
        struct cls_cgroup_head *head = rcu_dereference_bh(tp->root);
        u32 classid = task_get_classid(skb);

diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c
index 014cd3de7b5dd5ee8e33bce50625ffb72879230d..535668e1f748de516ec6daed8779b08c21f6f169 100644 (file)
--- a/net/sched/cls_flow.c
+++ b/net/sched/cls_flow.c
@@ -24,6 +24,7 @@
 #include <net/ip.h>
 #include <net/route.h>
 #include <net/flow_dissector.h>
+#include <net/tc_wrapper.h>
 
 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
 #include <net/netfilter/nf_conntrack.h>
@@ -292,8 +293,9 @@ static u32 flow_key_get(struct sk_buff *skb, int key, struct flow_keys *flow)
                          (1 << FLOW_KEY_NFCT_PROTO_SRC) |      \
                          (1 << FLOW_KEY_NFCT_PROTO_DST))
 
-static int flow_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                        struct tcf_result *res)
+TC_INDIRECT_SCOPE int flow_classify(struct sk_buff *skb,
+                                   const struct tcf_proto *tp,
+                                   struct tcf_result *res)
 {
        struct flow_head *head = rcu_dereference_bh(tp->root);
        struct flow_filter *f;

diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c
index 25bc57ee6ea1097c1a7a77703c389ab8294755af..0b15698b3531dd897cf17ea2208355ce1bdca81b 100644 (file)
--- a/net/sched/cls_flower.c
+++ b/net/sched/cls_flower.c
@@ -27,6 +27,7 @@
 #include <net/vxlan.h>
 #include <net/erspan.h>
 #include <net/gtp.h>
+#include <net/tc_wrapper.h>
 
 #include <net/dst.h>
 #include <net/dst_metadata.h>
@@ -305,8 +306,9 @@ static u16 fl_ct_info_to_flower_map[] = {
                                        TCA_FLOWER_KEY_CT_FLAGS_NEW,
 };
 
-static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                      struct tcf_result *res)
+TC_INDIRECT_SCOPE int fl_classify(struct sk_buff *skb,
+                                 const struct tcf_proto *tp,
+                                 struct tcf_result *res)
 {
        struct cls_fl_head *head = rcu_dereference_bh(tp->root);
        bool post_ct = tc_skb_cb(skb)->post_ct;

diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c
index a32351da968cd7cb6fea11daf491af314093e07b..ae9439a6c56c90f8f1711c8cde33cd967f1ef7e5 100644 (file)
--- a/net/sched/cls_fw.c
+++ b/net/sched/cls_fw.c
@@ -21,6 +21,7 @@
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
 #include <net/sch_generic.h>
+#include <net/tc_wrapper.h>
 
 #define HTSIZE 256
 
@@ -47,8 +48,9 @@ static u32 fw_hash(u32 handle)
        return handle % HTSIZE;
 }
 
-static int fw_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                      struct tcf_result *res)
+TC_INDIRECT_SCOPE int fw_classify(struct sk_buff *skb,
+                                 const struct tcf_proto *tp,
+                                 struct tcf_result *res)
 {
        struct fw_head *head = rcu_dereference_bh(tp->root);
        struct fw_filter *f;

diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c
index 39a5d9c170def16d7133930d30bf23125915269e..705f63da2c218ca700886e1baa7761064bef9fb5 100644 (file)
--- a/net/sched/cls_matchall.c
+++ b/net/sched/cls_matchall.c
@@ -12,6 +12,7 @@
 
 #include <net/sch_generic.h>
 #include <net/pkt_cls.h>
+#include <net/tc_wrapper.h>
 
 struct cls_mall_head {
        struct tcf_exts exts;
@@ -24,8 +25,9 @@ struct cls_mall_head {
        bool deleting;
 };
 
-static int mall_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                        struct tcf_result *res)
+TC_INDIRECT_SCOPE int mall_classify(struct sk_buff *skb,
+                                   const struct tcf_proto *tp,
+                                   struct tcf_result *res)
 {
        struct cls_mall_head *head = rcu_dereference_bh(tp->root);
 

diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c
index 9e43b929d4ca4e42a960315f143f4023f8b454d6..d0c53724d3e86bce003296950e8d5a71ff8227d2 100644 (file)
--- a/net/sched/cls_route.c
+++ b/net/sched/cls_route.c
@@ -17,6 +17,7 @@
 #include <net/netlink.h>
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
+#include <net/tc_wrapper.h>
 
 /*
  * 1. For now we assume that route tags < 256.
@@ -121,8 +122,9 @@ static inline int route4_hash_wild(void)
        return 0;                                               \
 }
 
-static int route4_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                          struct tcf_result *res)
+TC_INDIRECT_SCOPE int route4_classify(struct sk_buff *skb,
+                                     const struct tcf_proto *tp,
+                                     struct tcf_result *res)
 {
        struct route4_head *head = rcu_dereference_bh(tp->root);
        struct dst_entry *dst;

diff --git a/net/sched/cls_rsvp.c b/net/sched/cls_rsvp.c
index de1c1d4da59779ed0c81ab9ec88458225684adaf..03d8619bd9c6de1a6aaf0d3dff75a755c1f5d382 100644 (file)
--- a/net/sched/cls_rsvp.c
+++ b/net/sched/cls_rsvp.c
@@ -15,10 +15,12 @@
 #include <net/netlink.h>
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
+#include <net/tc_wrapper.h>
 
 #define RSVP_DST_LEN   1
 #define RSVP_ID                "rsvp"
 #define RSVP_OPS       cls_rsvp_ops
+#define RSVP_CLS       rsvp_classify
 
 #include "cls_rsvp.h"
 MODULE_LICENSE("GPL");

diff --git a/net/sched/cls_rsvp.h b/net/sched/cls_rsvp.h
index b00a7dbd0587403b16624aa05b6d8bafacbe4b7f..869efba9f8345ad98f14a3e3eed029b88ceeb52d 100644 (file)
--- a/net/sched/cls_rsvp.h
+++ b/net/sched/cls_rsvp.h
@@ -124,8 +124,8 @@ static inline unsigned int hash_src(__be32 *src)
                return r;                               \
 }
 
-static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                        struct tcf_result *res)
+TC_INDIRECT_SCOPE int RSVP_CLS(struct sk_buff *skb, const struct tcf_proto *tp,
+                              struct tcf_result *res)
 {
        struct rsvp_head *head = rcu_dereference_bh(tp->root);
        struct rsvp_session *s;
@@ -738,7 +738,7 @@ static void rsvp_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
 
 static struct tcf_proto_ops RSVP_OPS __read_mostly = {
        .kind           =       RSVP_ID,
-       .classify       =       rsvp_classify,
+       .classify       =       RSVP_CLS,
        .init           =       rsvp_init,
        .destroy        =       rsvp_destroy,
        .get            =       rsvp_get,

diff --git a/net/sched/cls_rsvp6.c b/net/sched/cls_rsvp6.c
index 64078846000ef51c49e45b1c2feb8b923e9e5d8f..e627cc32d6333213f5ef9e82a32bcffa71bb3f74 100644 (file)
--- a/net/sched/cls_rsvp6.c
+++ b/net/sched/cls_rsvp6.c
@@ -15,10 +15,12 @@
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
 #include <net/netlink.h>
+#include <net/tc_wrapper.h>
 
 #define RSVP_DST_LEN   4
 #define RSVP_ID                "rsvp6"
 #define RSVP_OPS       cls_rsvp6_ops
+#define RSVP_CLS rsvp6_classify
 
 #include "cls_rsvp.h"
 MODULE_LICENSE("GPL");

diff --git a/net/sched/cls_tcindex.c b/net/sched/cls_tcindex.c
index 1c9eeb98d826e42bc2ff2f169a36a04dac517b9d..eb0e9458e722e403a15fb6ebf4365a920870d7fa 100644 (file)
--- a/net/sched/cls_tcindex.c
+++ b/net/sched/cls_tcindex.c
@@ -16,6 +16,7 @@
 #include <net/netlink.h>
 #include <net/pkt_cls.h>
 #include <net/sch_generic.h>
+#include <net/tc_wrapper.h>
 
 /*
  * Passing parameters to the root seems to be done more awkwardly than really
@@ -98,9 +99,9 @@ static struct tcindex_filter_result *tcindex_lookup(struct tcindex_data *p,
        return NULL;
 }
 
-
-static int tcindex_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                           struct tcf_result *res)
+TC_INDIRECT_SCOPE int tcindex_classify(struct sk_buff *skb,
+                                      const struct tcf_proto *tp,
+                                      struct tcf_result *res)
 {
        struct tcindex_data *p = rcu_dereference_bh(tp->root);
        struct tcindex_filter_result *f;

diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c
index 34d25f7a0687a7f24f49554e7187a62ae8945781..4e2e269f121f8a301368b9783753e055f5af6a4e 100644 (file)
--- a/net/sched/cls_u32.c
+++ b/net/sched/cls_u32.c
@@ -39,6 +39,7 @@
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
 #include <linux/idr.h>
+#include <net/tc_wrapper.h>
 
 struct tc_u_knode {
        struct tc_u_knode __rcu *next;
@@ -100,8 +101,9 @@ static inline unsigned int u32_hash_fold(__be32 key,
        return h;
 }
 
-static int u32_classify(struct sk_buff *skb, const struct tcf_proto *tp,
-                       struct tcf_result *res)
+TC_INDIRECT_SCOPE int u32_classify(struct sk_buff *skb,
+                                  const struct tcf_proto *tp,
+                                  struct tcf_result *res)
 {
        struct {
                struct tc_u_knode *knode;