netfilter: nf_tables: add direct calls for all builtin expressions

With CONFIG_RETPOLINE its faster to add an if (ptr == &foo_func)
check and and use direct calls for all the built-in expressions.

~15% improvement in pathological cases.

checkpatch doesn't like the X macro due to the embedded return statement,
but the macro has a very limited scope so I don't think its a problem.

I would like to avoid bugs of the form
  If (e->ops->eval == (unsigned long)nft_foo_eval)
	 nft_bar_eval();

and open-coded if ()/else if()/else cascade, thus the macro.

Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>

diff --git a/include/net/netfilter/nf_tables_core.h b/include/net/netfilter/nf_tables_core.h
index 2046d10..7281895 100644 (file)
--- a/include/net/netfilter/nf_tables_core.h
+++ b/include/net/netfilter/nf_tables_core.h
@@ -80,6 +80,22 @@ struct nft_regs;
 struct nft_pktinfo;
 void nft_meta_get_eval(const struct nft_expr *expr,
                       struct nft_regs *regs, const struct nft_pktinfo *pkt);
+void nft_cmp_eval(const struct nft_expr *expr,
+                 struct nft_regs *regs, const struct nft_pktinfo *pkt);
 void nft_lookup_eval(const struct nft_expr *expr,
                     struct nft_regs *regs, const struct nft_pktinfo *pkt);
+void nft_payload_eval(const struct nft_expr *expr,
+                     struct nft_regs *regs, const struct nft_pktinfo *pkt);
+void nft_immediate_eval(const struct nft_expr *expr,
+                       struct nft_regs *regs, const struct nft_pktinfo *pkt);
+void nft_bitwise_eval(const struct nft_expr *expr,
+                     struct nft_regs *regs, const struct nft_pktinfo *pkt);
+void nft_range_eval(const struct nft_expr *expr,
+                   struct nft_regs *regs, const struct nft_pktinfo *pkt);
+void nft_byteorder_eval(const struct nft_expr *expr,
+                       struct nft_regs *regs, const struct nft_pktinfo *pkt);
+void nft_dynset_eval(const struct nft_expr *expr,
+                    struct nft_regs *regs, const struct nft_pktinfo *pkt);
+void nft_rt_get_eval(const struct nft_expr *expr,
+                    struct nft_regs *regs, const struct nft_pktinfo *pkt);
 #endif /* _NET_NF_TABLES_CORE_H */

diff --git a/net/netfilter/nf_tables_core.c b/net/netfilter/nf_tables_core.c
index a505002..2a00aef 100644 (file)
--- a/net/netfilter/nf_tables_core.c
+++ b/net/netfilter/nf_tables_core.c
@@ -124,14 +124,25 @@ static void expr_call_ops_eval(const struct nft_expr *expr,
                               struct nft_regs *regs,
                               struct nft_pktinfo *pkt)
 {
+#ifdef CONFIG_RETPOLINE
        unsigned long e = (unsigned long)expr->ops->eval;
-
-       if (e == (unsigned long)nft_meta_get_eval)
-               nft_meta_get_eval(expr, regs, pkt);
-       else if (e == (unsigned long)nft_lookup_eval)
-               nft_lookup_eval(expr, regs, pkt);
-       else
-               expr->ops->eval(expr, regs, pkt);
+#define X(e, fun) \
+       do { if ((e) == (unsigned long)(fun)) \
+               return fun(expr, regs, pkt); } while (0)
+
+       X(e, nft_payload_eval);
+       X(e, nft_cmp_eval);
+       X(e, nft_meta_get_eval);
+       X(e, nft_lookup_eval);
+       X(e, nft_range_eval);
+       X(e, nft_immediate_eval);
+       X(e, nft_byteorder_eval);
+       X(e, nft_dynset_eval);
+       X(e, nft_rt_get_eval);
+       X(e, nft_bitwise_eval);
+#undef  X
+#endif /* CONFIG_RETPOLINE */
+       expr->ops->eval(expr, regs, pkt);
 }
 
 unsigned int

diff --git a/net/netfilter/nft_bitwise.c b/net/netfilter/nft_bitwise.c
index fff8073..2c75b9e 100644 (file)
--- a/net/netfilter/nft_bitwise.c
+++ b/net/netfilter/nft_bitwise.c
@@ -25,9 +25,8 @@ struct nft_bitwise {
        struct nft_data         xor;
 };
 
-static void nft_bitwise_eval(const struct nft_expr *expr,
-                            struct nft_regs *regs,
-                            const struct nft_pktinfo *pkt)
+void nft_bitwise_eval(const struct nft_expr *expr,
+                     struct nft_regs *regs, const struct nft_pktinfo *pkt)
 {
        const struct nft_bitwise *priv = nft_expr_priv(expr);
        const u32 *src = &regs->data[priv->sreg];

diff --git a/net/netfilter/nft_byteorder.c b/net/netfilter/nft_byteorder.c
index 13d4e42..19dbc34 100644 (file)
--- a/net/netfilter/nft_byteorder.c
+++ b/net/netfilter/nft_byteorder.c
@@ -26,9 +26,9 @@ struct nft_byteorder {
        u8                      size;
 };
 
-static void nft_byteorder_eval(const struct nft_expr *expr,
-                              struct nft_regs *regs,
-                              const struct nft_pktinfo *pkt)
+void nft_byteorder_eval(const struct nft_expr *expr,
+                       struct nft_regs *regs,
+                       const struct nft_pktinfo *pkt)
 {
        const struct nft_byteorder *priv = nft_expr_priv(expr);
        u32 *src = &regs->data[priv->sreg];

diff --git a/net/netfilter/nft_cmp.c b/net/netfilter/nft_cmp.c
index 79d48c1..f9f1fa6 100644 (file)
--- a/net/netfilter/nft_cmp.c
+++ b/net/netfilter/nft_cmp.c
@@ -24,9 +24,9 @@ struct nft_cmp_expr {
        enum nft_cmp_ops        op:8;
 };
 
-static void nft_cmp_eval(const struct nft_expr *expr,
-                        struct nft_regs *regs,
-                        const struct nft_pktinfo *pkt)
+void nft_cmp_eval(const struct nft_expr *expr,
+                 struct nft_regs *regs,
+                 const struct nft_pktinfo *pkt)
 {
        const struct nft_cmp_expr *priv = nft_expr_priv(expr);
        int d;

diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c
index 07d4efd..9658493 100644 (file)
--- a/net/netfilter/nft_dynset.c
+++ b/net/netfilter/nft_dynset.c
@@ -62,9 +62,8 @@ err1:
        return NULL;
 }
 
-static void nft_dynset_eval(const struct nft_expr *expr,
-                           struct nft_regs *regs,
-                           const struct nft_pktinfo *pkt)
+void nft_dynset_eval(const struct nft_expr *expr,
+                    struct nft_regs *regs, const struct nft_pktinfo *pkt)
 {
        const struct nft_dynset *priv = nft_expr_priv(expr);
        struct nft_set *set = priv->set;

diff --git a/net/netfilter/nft_immediate.c b/net/netfilter/nft_immediate.c
index 0777a93..3e5ed78 100644 (file)
--- a/net/netfilter/nft_immediate.c
+++ b/net/netfilter/nft_immediate.c
@@ -17,9 +17,9 @@
 #include <net/netfilter/nf_tables_core.h>
 #include <net/netfilter/nf_tables.h>
 
-static void nft_immediate_eval(const struct nft_expr *expr,
-                              struct nft_regs *regs,
-                              const struct nft_pktinfo *pkt)
+void nft_immediate_eval(const struct nft_expr *expr,
+                       struct nft_regs *regs,
+                       const struct nft_pktinfo *pkt)
 {
        const struct nft_immediate_expr *priv = nft_expr_priv(expr);
 

diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c
index e110b0e..54e15de 100644 (file)
--- a/net/netfilter/nft_payload.c
+++ b/net/netfilter/nft_payload.c
@@ -70,9 +70,9 @@ nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u8 offset, u8 len)
        return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0;
 }
 
-static void nft_payload_eval(const struct nft_expr *expr,
-                            struct nft_regs *regs,
-                            const struct nft_pktinfo *pkt)
+void nft_payload_eval(const struct nft_expr *expr,
+                     struct nft_regs *regs,
+                     const struct nft_pktinfo *pkt)
 {
        const struct nft_payload *priv = nft_expr_priv(expr);
        const struct sk_buff *skb = pkt->skb;

diff --git a/net/netfilter/nft_range.c b/net/netfilter/nft_range.c
index cedb96c..529ac8a 100644 (file)
--- a/net/netfilter/nft_range.c
+++ b/net/netfilter/nft_range.c
@@ -23,9 +23,8 @@ struct nft_range_expr {
        enum nft_range_ops      op:8;
 };
 
-static void nft_range_eval(const struct nft_expr *expr,
-                        struct nft_regs *regs,
-                        const struct nft_pktinfo *pkt)
+void nft_range_eval(const struct nft_expr *expr,
+                   struct nft_regs *regs, const struct nft_pktinfo *pkt)
 {
        const struct nft_range_expr *priv = nft_expr_priv(expr);
        int d1, d2;

diff --git a/net/netfilter/nft_rt.c b/net/netfilter/nft_rt.c
index f35fa33..c48daed 100644 (file)
--- a/net/netfilter/nft_rt.c
+++ b/net/netfilter/nft_rt.c
@@ -53,9 +53,9 @@ static u16 get_tcpmss(const struct nft_pktinfo *pkt, const struct dst_entry *skb
        return mtu - minlen;
 }
 
-static void nft_rt_get_eval(const struct nft_expr *expr,
-                           struct nft_regs *regs,
-                           const struct nft_pktinfo *pkt)
+void nft_rt_get_eval(const struct nft_expr *expr,
+                    struct nft_regs *regs,
+                    const struct nft_pktinfo *pkt)
 {
        const struct nft_rt *priv = nft_expr_priv(expr);
        const struct sk_buff *skb = pkt->skb;