netfilter: x_tables: pack percpu counter allocations

commit ae0ac0ed6fcf5af3be0f63eb935f483f44a402d2 upstream.

instead of allocating each xt_counter individually, allocate 4k chunks
and then use these for counter allocation requests.

This should speed up rule evaluation by increasing data locality,
also speeds up ruleset loading because we reduce calls to the percpu
allocator.

As Eric points out we can't use PAGE_SIZE, page_allocator would fail on
arches with 64k page size.

Suggested-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/include/linux/netfilter/x_tables.h b/include/linux/netfilter/x_tables.h
index 58f37f8a3d8b7ca07baca70cafce1f7bbea26151..9bfeb88fb940cf6c1b17e817de1a66e5c740a171 100644 (file)
--- a/include/linux/netfilter/x_tables.h
+++ b/include/linux/netfilter/x_tables.h
@@ -375,8 +375,13 @@ static inline unsigned long ifname_compare_aligned(const char *_a,
        return ret;
 }
 
+struct xt_percpu_counter_alloc_state {
+       unsigned int off;
+       const char __percpu *mem;
+};
 
-bool xt_percpu_counter_alloc(struct xt_counters *counters);
+bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
+                            struct xt_counters *counter);
 void xt_percpu_counter_free(struct xt_counters *cnt);
 
 static inline struct xt_counters *

diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c
index 6f116d384c013349972190e6cbd5e7c7c8eaf394..d35815e5967b567356fa5db1f86f1be9ad30ac03 100644 (file)
--- a/net/ipv4/netfilter/arp_tables.c
+++ b/net/ipv4/netfilter/arp_tables.c
@@ -419,13 +419,14 @@ static inline int check_target(struct arpt_entry *e, const char *name)
 }
 
 static inline int
-find_check_entry(struct arpt_entry *e, const char *name, unsigned int size)
+find_check_entry(struct arpt_entry *e, const char *name, unsigned int size,
+                struct xt_percpu_counter_alloc_state *alloc_state)
 {
        struct xt_entry_target *t;
        struct xt_target *target;
        int ret;
 
-       if (!xt_percpu_counter_alloc(&e->counters))
+       if (!xt_percpu_counter_alloc(alloc_state, &e->counters))
                return -ENOMEM;
 
        t = arpt_get_target(e);
@@ -533,6 +534,7 @@ static inline void cleanup_entry(struct arpt_entry *e)
 static int translate_table(struct xt_table_info *newinfo, void *entry0,
                           const struct arpt_replace *repl)
 {
+       struct xt_percpu_counter_alloc_state alloc_state = { 0 };
        struct arpt_entry *iter;
        unsigned int *offsets;
        unsigned int i;
@@ -595,7 +597,8 @@ static int translate_table(struct xt_table_info *newinfo, void *entry0,
        /* Finally, each sanity check must pass */
        i = 0;
        xt_entry_foreach(iter, entry0, newinfo->size) {
-               ret = find_check_entry(iter, repl->name, repl->size);
+               ret = find_check_entry(iter, repl->name, repl->size,
+                                      &alloc_state);
                if (ret != 0)
                        break;
                ++i;

diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c
index 021ddf17d000336edaec8b1259e7b8d308421177..e78f6521823f748be76ecd2eaed2b4e5a86c6407 100644 (file)
--- a/net/ipv4/netfilter/ip_tables.c
+++ b/net/ipv4/netfilter/ip_tables.c
@@ -540,7 +540,8 @@ static int check_target(struct ipt_entry *e, struct net *net, const char *name)
 
 static int
 find_check_entry(struct ipt_entry *e, struct net *net, const char *name,
-                unsigned int size)
+                unsigned int size,
+                struct xt_percpu_counter_alloc_state *alloc_state)
 {
        struct xt_entry_target *t;
        struct xt_target *target;
@@ -549,7 +550,7 @@ find_check_entry(struct ipt_entry *e, struct net *net, const char *name,
        struct xt_mtchk_param mtpar;
        struct xt_entry_match *ematch;
 
-       if (!xt_percpu_counter_alloc(&e->counters))
+       if (!xt_percpu_counter_alloc(alloc_state, &e->counters))
                return -ENOMEM;
 
        j = 0;
@@ -685,6 +686,7 @@ static int
 translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0,
                const struct ipt_replace *repl)
 {
+       struct xt_percpu_counter_alloc_state alloc_state = { 0 };
        struct ipt_entry *iter;
        unsigned int *offsets;
        unsigned int i;
@@ -744,7 +746,8 @@ translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0,
        /* Finally, each sanity check must pass */
        i = 0;
        xt_entry_foreach(iter, entry0, newinfo->size) {
-               ret = find_check_entry(iter, net, repl->name, repl->size);
+               ret = find_check_entry(iter, net, repl->name, repl->size,
+                                      &alloc_state);
                if (ret != 0)
                        break;
                ++i;

diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c
index d70b302b2f21a45d4381ce671695d6a23d0dce63..e26becc9a43dab4e656ae73703016df96441ddd9 100644 (file)
--- a/net/ipv6/netfilter/ip6_tables.c
+++ b/net/ipv6/netfilter/ip6_tables.c
@@ -570,7 +570,8 @@ static int check_target(struct ip6t_entry *e, struct net *net, const char *name)
 
 static int
 find_check_entry(struct ip6t_entry *e, struct net *net, const char *name,
-                unsigned int size)
+                unsigned int size,
+                struct xt_percpu_counter_alloc_state *alloc_state)
 {
        struct xt_entry_target *t;
        struct xt_target *target;
@@ -579,7 +580,7 @@ find_check_entry(struct ip6t_entry *e, struct net *net, const char *name,
        struct xt_mtchk_param mtpar;
        struct xt_entry_match *ematch;
 
-       if (!xt_percpu_counter_alloc(&e->counters))
+       if (!xt_percpu_counter_alloc(alloc_state, &e->counters))
                return -ENOMEM;
 
        j = 0;
@@ -713,6 +714,7 @@ static int
 translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0,
                const struct ip6t_replace *repl)
 {
+       struct xt_percpu_counter_alloc_state alloc_state = { 0 };
        struct ip6t_entry *iter;
        unsigned int *offsets;
        unsigned int i;
@@ -772,7 +774,8 @@ translate_table(struct net *net, struct xt_table_info *newinfo, void *entry0,
        /* Finally, each sanity check must pass */
        i = 0;
        xt_entry_foreach(iter, entry0, newinfo->size) {
-               ret = find_check_entry(iter, net, repl->name, repl->size);
+               ret = find_check_entry(iter, net, repl->name, repl->size,
+                                      &alloc_state);
                if (ret != 0)
                        break;
                ++i;

diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c
index 9736743e747c989e79a4334c2e18a47053305d9b..ac26b71d900e4442d01205c887653268b6e923ec 100644 (file)
--- a/net/netfilter/x_tables.c
+++ b/net/netfilter/x_tables.c
@@ -39,6 +39,8 @@ MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Harald Welte <laforge@netfilter.org>");
 MODULE_DESCRIPTION("{ip,ip6,arp,eb}_tables backend module");
 
+#define XT_PCPU_BLOCK_SIZE 4096
+
 struct compat_delta {
        unsigned int offset; /* offset in kernel */
        int delta; /* delta in 32bit user land */
@@ -1622,6 +1624,7 @@ EXPORT_SYMBOL_GPL(xt_proto_fini);
 /**
  * xt_percpu_counter_alloc - allocate x_tables rule counter
  *
+ * @state: pointer to xt_percpu allocation state
  * @counter: pointer to counter struct inside the ip(6)/arpt_entry struct
  *
  * On SMP, the packet counter [ ip(6)t_entry->counters.pcnt ] will then
@@ -1630,21 +1633,34 @@ EXPORT_SYMBOL_GPL(xt_proto_fini);
  * Rule evaluation needs to use xt_get_this_cpu_counter() helper
  * to fetch the real percpu counter.
  *
+ * To speed up allocation and improve data locality, a 4kb block is
+ * allocated.
+ *
+ * xt_percpu_counter_alloc_state contains the base address of the
+ * allocated page and the current sub-offset.
+ *
  * returns false on error.
  */
-bool xt_percpu_counter_alloc(struct xt_counters *counter)
+bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
+                            struct xt_counters *counter)
 {
-       void __percpu *res;
+       BUILD_BUG_ON(XT_PCPU_BLOCK_SIZE < (sizeof(*counter) * 2));
 
        if (nr_cpu_ids <= 1)
                return true;
 
-       res = __alloc_percpu(sizeof(struct xt_counters),
-                            sizeof(struct xt_counters));
-       if (!res)
-               return false;
-
-       counter->pcnt = (__force unsigned long)res;
+       if (!state->mem) {
+               state->mem = __alloc_percpu(XT_PCPU_BLOCK_SIZE,
+                                           XT_PCPU_BLOCK_SIZE);
+               if (!state->mem)
+                       return false;
+       }
+       counter->pcnt = (__force unsigned long)(state->mem + state->off);
+       state->off += sizeof(*counter);
+       if (state->off > (XT_PCPU_BLOCK_SIZE - sizeof(*counter))) {
+               state->mem = NULL;
+               state->off = 0;
+       }
        return true;
 }
 EXPORT_SYMBOL_GPL(xt_percpu_counter_alloc);
@@ -1653,7 +1669,7 @@ void xt_percpu_counter_free(struct xt_counters *counters)
 {
        unsigned long pcnt = counters->pcnt;
 
-       if (nr_cpu_ids > 1)
+       if (nr_cpu_ids > 1 && (pcnt & (XT_PCPU_BLOCK_SIZE - 1)) == 0)
                free_percpu((void __percpu *)pcnt);
 }
 EXPORT_SYMBOL_GPL(xt_percpu_counter_free);