lib/find_bit: optimize find_next_bit() functions

Over the past couple years, the function _find_next_bit() was extended
with parameters that modify its behavior to implement and- zero- and le-
flavors. The parameters are passed at compile time, but current design
prevents a compiler from optimizing out the conditionals.

As find_next_bit() API grows, I expect that more parameters will be added.
Current design would require more conditional code in _find_next_bit(),
which would bloat the helper even more and make it barely readable.

This patch replaces _find_next_bit() with a macro FIND_NEXT_BIT, and adds
a set of wrappers, so that the compile-time optimizations become possible.

The common logic is moved to the new macro, and all flavors may be
generated by providing a FETCH macro parameter, like in this example:

  #define FIND_NEXT_BIT(FETCH, MUNGE, size, start) ...

  find_next_xornot_and_bit(addr1, addr2, addr3, size, start)
  {
	return FIND_NEXT_BIT(addr1[idx] ^ ~addr2[idx] & addr3[idx],
				/* nop */, size, start);
  }

The FETCH may be of any complexity, as soon as it only refers the bitmap(s)
and an iterator idx.

MUNGE is here to support _le code generation for BE builds. May be
empty.

I ran find_bit_benchmark 16 times on top of 6.0-rc2 and 16 times on top
of 6.0-rc2 + this series. The results for kvm/x86_64 are:

                      v6.0-rc2  Optimized       Difference  Z-score
Random dense bitmap         ns         ns        ns      %
find_next_bit:          787735     670546    117189   14.9     3.97
find_next_zero_bit:     777492     664208    113284   14.6    10.51
find_last_bit:          830925     687573    143352   17.3     2.35
find_first_bit:        3874366    3306635    567731   14.7     1.84
find_first_and_bit:   40677125   37739887   2937238    7.2     1.36
find_next_and_bit:      347865     304456     43409   12.5     1.35

Random sparse bitmap
find_next_bit:           19816      14021      5795   29.2     6.10
find_next_zero_bit:    1318901    1223794     95107    7.2     1.41
find_last_bit:           14573      13514      1059    7.3     6.92
find_first_bit:        1313321    1249024     64297    4.9     1.53
find_first_and_bit:       8921       8098       823    9.2     4.56
find_next_and_bit:        9796       7176      2620   26.7     5.39

Where the statistics is significant (z-score > 3), the improvement
is ~15%.

According to the bloat-o-meter, the Image size is 10-11K less:

x86_64/defconfig:
add/remove: 32/14 grow/shrink: 61/782 up/down: 6344/-16521 (-10177)

arm64/defconfig:
add/remove: 3/2 grow/shrink: 50/714 up/down: 608/-11556 (-10948)

Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Yury Norov <yury.norov@gmail.com>

diff --git a/include/linux/find.h b/include/linux/find.h
index 2464bff5de04ce3402949daccc5bfe5d1e51d806..dead6f53a97bcb9040a719897f8e3210e7bab07f 100644 (file)
--- a/include/linux/find.h
+++ b/include/linux/find.h
@@ -8,9 +8,12 @@
 
 #include <linux/bitops.h>
 
-extern unsigned long _find_next_bit(const unsigned long *addr1,
-               const unsigned long *addr2, unsigned long nbits,
-               unsigned long start, unsigned long invert, unsigned long le);
+unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits,
+                               unsigned long start);
+unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
+                                       unsigned long nbits, unsigned long start);
+unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
+                                        unsigned long start);
 extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size);
 extern unsigned long _find_first_and_bit(const unsigned long *addr1,
                                         const unsigned long *addr2, unsigned long size);
@@ -19,6 +22,10 @@ extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long siz
 
 #ifdef __BIG_ENDIAN
 unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size);
+unsigned long _find_next_zero_bit_le(const  unsigned long *addr, unsigned
+                                       long size, unsigned long offset);
+unsigned long _find_next_bit_le(const unsigned long *addr, unsigned
+                               long size, unsigned long offset);
 #endif
 
 #ifndef find_next_bit
@@ -45,7 +52,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
                return val ? __ffs(val) : size;
        }
 
-       return _find_next_bit(addr, NULL, size, offset, 0UL, 0);
+       return _find_next_bit(addr, size, offset);
 }
 #endif
 
@@ -75,7 +82,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1,
                return val ? __ffs(val) : size;
        }
 
-       return _find_next_bit(addr1, addr2, size, offset, 0UL, 0);
+       return _find_next_and_bit(addr1, addr2, size, offset);
 }
 #endif
 
@@ -103,7 +110,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
                return val == ~0UL ? size : ffz(val);
        }
 
-       return _find_next_bit(addr, NULL, size, offset, ~0UL, 0);
+       return _find_next_zero_bit(addr, size, offset);
 }
 #endif
 
@@ -251,7 +258,7 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned
                return val == ~0UL ? size : ffz(val);
        }
 
-       return _find_next_bit(addr, NULL, size, offset, ~0UL, 1);
+       return _find_next_zero_bit_le(addr, size, offset);
 }
 #endif
 
@@ -284,7 +291,7 @@ unsigned long find_next_bit_le(const void *addr, unsigned
                return val ? __ffs(val) : size;
        }
 
-       return _find_next_bit(addr, NULL, size, offset, 0UL, 1);
+       return _find_next_bit_le(addr, size, offset);
 }
 #endif
 

diff --git a/lib/find_bit.c b/lib/find_bit.c
index 61ec2992938bfe38756d87016fd05e34e3302bbb..d00ee23ab65736a1b33da458e95d0b220bbeca17 100644 (file)
--- a/lib/find_bit.c
+++ b/lib/find_bit.c
@@ -40,57 +40,33 @@
        sz;                                                                     \
 })
 
-#if !defined(find_next_bit) || !defined(find_next_zero_bit) ||                 \
-       !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) ||        \
-       !defined(find_next_and_bit)
 /*
- * This is a common helper function for find_next_bit, find_next_zero_bit, and
- * find_next_and_bit. The differences are:
- *  - The "invert" argument, which is XORed with each fetched word before
- *    searching it for one bits.
- *  - The optional "addr2", which is anded with "addr1" if present.
+ * Common helper for find_next_bit() function family
+ * @FETCH: The expression that fetches and pre-processes each word of bitmap(s)
+ * @MUNGE: The expression that post-processes a word containing found bit (may be empty)
+ * @size: The bitmap size in bits
+ * @start: The bitnumber to start searching at
  */
-unsigned long _find_next_bit(const unsigned long *addr1,
-               const unsigned long *addr2, unsigned long nbits,
-               unsigned long start, unsigned long invert, unsigned long le)
-{
-       unsigned long tmp, mask;
-
-       if (unlikely(start >= nbits))
-               return nbits;
-
-       tmp = addr1[start / BITS_PER_LONG];
-       if (addr2)
-               tmp &= addr2[start / BITS_PER_LONG];
-       tmp ^= invert;
-
-       /* Handle 1st word. */
-       mask = BITMAP_FIRST_WORD_MASK(start);
-       if (le)
-               mask = swab(mask);
-
-       tmp &= mask;
-
-       start = round_down(start, BITS_PER_LONG);
-
-       while (!tmp) {
-               start += BITS_PER_LONG;
-               if (start >= nbits)
-                       return nbits;
-
-               tmp = addr1[start / BITS_PER_LONG];
-               if (addr2)
-                       tmp &= addr2[start / BITS_PER_LONG];
-               tmp ^= invert;
-       }
-
-       if (le)
-               tmp = swab(tmp);
-
-       return min(start + __ffs(tmp), nbits);
-}
-EXPORT_SYMBOL(_find_next_bit);
-#endif
+#define FIND_NEXT_BIT(FETCH, MUNGE, size, start)                               \
+({                                                                             \
+       unsigned long mask, idx, tmp, sz = (size), __start = (start);           \
+                                                                               \
+       if (unlikely(__start >= sz))                                            \
+               goto out;                                                       \
+                                                                               \
+       mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start));                          \
+       idx = __start / BITS_PER_LONG;                                          \
+                                                                               \
+       for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) {                       \
+               if ((idx + 1) * BITS_PER_LONG >= sz)                            \
+                       goto out;                                               \
+               idx++;                                                          \
+       }                                                                       \
+                                                                               \
+       sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz);                  \
+out:                                                                           \
+       sz;                                                                     \
+})
 
 #ifndef find_first_bit
 /*
@@ -127,6 +103,32 @@ unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size
 EXPORT_SYMBOL(_find_first_zero_bit);
 #endif
 
+#ifndef find_next_bit
+unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start)
+{
+       return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start);
+}
+EXPORT_SYMBOL(_find_next_bit);
+#endif
+
+#ifndef find_next_and_bit
+unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
+                                       unsigned long nbits, unsigned long start)
+{
+       return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start);
+}
+EXPORT_SYMBOL(_find_next_and_bit);
+#endif
+
+#ifndef find_next_zero_bit
+unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
+                                        unsigned long start)
+{
+       return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start);
+}
+EXPORT_SYMBOL(_find_next_zero_bit);
+#endif
+
 #ifndef find_last_bit
 unsigned long _find_last_bit(const unsigned long *addr, unsigned long size)
 {
@@ -175,4 +177,23 @@ EXPORT_SYMBOL(_find_first_zero_bit_le);
 
 #endif
 
+#ifndef find_next_zero_bit_le
+unsigned long _find_next_zero_bit_le(const unsigned long *addr,
+                                       unsigned long size, unsigned long offset)
+{
+       return FIND_NEXT_BIT(~addr[idx], swab, size, offset);
+}
+EXPORT_SYMBOL(_find_next_zero_bit_le);
+#endif
+
+#ifndef find_next_bit_le
+unsigned long _find_next_bit_le(const unsigned long *addr,
+                               unsigned long size, unsigned long offset)
+{
+       return FIND_NEXT_BIT(addr[idx], swab, size, offset);
+}
+EXPORT_SYMBOL(_find_next_bit_le);
+
+#endif
+
 #endif /* __BIG_ENDIAN */