This patch drops the ZeroBehavior parameter from bit counting
functions like countLeadingZeros. ZeroBehavior specifies the behavior
when the input to count{Leading,Trailing}Zeros is zero and when the
input to count{Leading,Trailing}Ones is all ones.
ZeroBehavior was first introduced on May 24, 2013 in commit
eb91eac9fb866ab1243366d2e238b9961895612d. While that patch did not
state the intention, I would guess ZeroBehavior was for performance
reasons. The x86 machines around that time required a conditional
branch to implement countLeadingZero<uint32_t> that returns the 32 on
zero:
test edi, edi
je .LBB0_2
bsr eax, edi
xor eax, 31
.LBB1_2:
mov eax, 32
That is, we can remove the conditional branch if we don't care about
the behavior on zero.
IIUC, Intel's Haswell architecture, launched on June 4, 2013,
introduced several bit manipulation instructions, including lzcnt and
tzcnt, which eliminated the need for the conditional branch.
I think it's time to retire ZeroBehavior as its utility is very
limited. If you care about compilation speed, you should build LLVM
with an appropriate -march= to take advantage of lzcnt and tzcnt.
Even if not, modern host compilers should be able to optimize away
quite a few conditional branches because the input is often known to
be nonzero from dominating conditional branches.
In this iteration, I've moved the forward declarations of
_BitScanForward outside the llvm space to fix builds on Windows.
Differential Revision: https://reviews.llvm.org/D141798
#include "llvm/Support/Compiler.h"
#include <cstdint>
#include <cstring>
+#include <limits>
#include <type_traits>
+#ifdef _MSC_VER
+// Declare these intrinsics manually rather including intrin.h. It's very
+// expensive, and bit.h is popular via MathExtras.h.
+// #include <intrin.h>
+extern "C" {
+unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
+unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
+unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
+unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
+}
+#endif
+
namespace llvm {
// This implementation of bit_cast is different from the C++20 one in two ways:
}
namespace detail {
+template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
+ static unsigned count(T Val) {
+ if (!Val)
+ return std::numeric_limits<T>::digits;
+ if (Val & 0x1)
+ return 0;
+
+ // Bisection method.
+ unsigned ZeroBits = 0;
+ T Shift = std::numeric_limits<T>::digits >> 1;
+ T Mask = std::numeric_limits<T>::max() >> Shift;
+ while (Shift) {
+ if ((Val & Mask) == 0) {
+ Val >>= Shift;
+ ZeroBits |= Shift;
+ }
+ Shift >>= 1;
+ Mask >>= Shift;
+ }
+ return ZeroBits;
+ }
+};
+
+#if defined(__GNUC__) || defined(_MSC_VER)
+template <typename T> struct TrailingZerosCounter<T, 4> {
+ static unsigned count(T Val) {
+ if (Val == 0)
+ return 32;
+
+#if __has_builtin(__builtin_ctz) || defined(__GNUC__)
+ return __builtin_ctz(Val);
+#elif defined(_MSC_VER)
+ unsigned long Index;
+ _BitScanForward(&Index, Val);
+ return Index;
+#endif
+ }
+};
+
+#if !defined(_MSC_VER) || defined(_M_X64)
+template <typename T> struct TrailingZerosCounter<T, 8> {
+ static unsigned count(T Val) {
+ if (Val == 0)
+ return 64;
+
+#if __has_builtin(__builtin_ctzll) || defined(__GNUC__)
+ return __builtin_ctzll(Val);
+#elif defined(_MSC_VER)
+ unsigned long Index;
+ _BitScanForward64(&Index, Val);
+ return Index;
+#endif
+ }
+};
+#endif
+#endif
+} // namespace detail
+
+/// Count number of 0's from the least significant bit to the most
+/// stopping at the first 1.
+///
+/// Only unsigned integral types are allowed.
+///
+/// Returns std::numeric_limits<T>::digits on an input of 0.
+template <typename T> int countr_zero(T Val) {
+ static_assert(std::is_unsigned_v<T>,
+ "Only unsigned integral types are allowed.");
+ return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val);
+}
+
+namespace detail {
+template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
+ static unsigned count(T Val) {
+ if (!Val)
+ return std::numeric_limits<T>::digits;
+
+ // Bisection method.
+ unsigned ZeroBits = 0;
+ for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
+ T Tmp = Val >> Shift;
+ if (Tmp)
+ Val = Tmp;
+ else
+ ZeroBits |= Shift;
+ }
+ return ZeroBits;
+ }
+};
+
+#if defined(__GNUC__) || defined(_MSC_VER)
+template <typename T> struct LeadingZerosCounter<T, 4> {
+ static unsigned count(T Val) {
+ if (Val == 0)
+ return 32;
+
+#if __has_builtin(__builtin_clz) || defined(__GNUC__)
+ return __builtin_clz(Val);
+#elif defined(_MSC_VER)
+ unsigned long Index;
+ _BitScanReverse(&Index, Val);
+ return Index ^ 31;
+#endif
+ }
+};
+
+#if !defined(_MSC_VER) || defined(_M_X64)
+template <typename T> struct LeadingZerosCounter<T, 8> {
+ static unsigned count(T Val) {
+ if (Val == 0)
+ return 64;
+
+#if __has_builtin(__builtin_clzll) || defined(__GNUC__)
+ return __builtin_clzll(Val);
+#elif defined(_MSC_VER)
+ unsigned long Index;
+ _BitScanReverse64(&Index, Val);
+ return Index ^ 63;
+#endif
+ }
+};
+#endif
+#endif
+} // namespace detail
+
+/// Count number of 0's from the most significant bit to the least
+/// stopping at the first 1.
+///
+/// Only unsigned integral types are allowed.
+///
+/// Returns std::numeric_limits<T>::digits on an input of 0.
+template <typename T> int countl_zero(T Val) {
+ static_assert(std::is_unsigned_v<T>,
+ "Only unsigned integral types are allowed.");
+ return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val);
+}
+
+/// Count the number of ones from the most significant bit to the first
+/// zero bit.
+///
+/// Ex. countl_one(0xFF0FFF00) == 8.
+/// Only unsigned integral types are allowed.
+///
+/// Returns std::numeric_limits<T>::digits on an input of all ones.
+template <typename T> int countl_one(T Value) {
+ static_assert(std::is_unsigned_v<T>,
+ "Only unsigned integral types are allowed.");
+ return llvm::countl_zero<T>(~Value);
+}
+
+/// Count the number of ones from the least significant bit to the first
+/// zero bit.
+///
+/// Ex. countr_one(0x00FF00FF) == 8.
+/// Only unsigned integral types are allowed.
+///
+/// Returns std::numeric_limits<T>::digits on an input of all ones.
+template <typename T> int countr_one(T Value) {
+ static_assert(std::is_unsigned_v<T>,
+ "Only unsigned integral types are allowed.");
+ return llvm::countr_zero<T>(~Value);
+}
+
+namespace detail {
template <typename T, std::size_t SizeOfT> struct PopulationCounter {
static int count(T Value) {
// Generic version, forward to 32 bits.
#include <limits>
#include <type_traits>
-#ifdef _MSC_VER
-// Declare these intrinsics manually rather including intrin.h. It's very
-// expensive, and MathExtras.h is popular.
-// #include <intrin.h>
-extern "C" {
-unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
-unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
-unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
-unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
-}
-#endif
-
namespace llvm {
/// The behavior an operation has on an input of 0.
phif = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622
} // namespace numbers
-namespace detail {
-template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
- static unsigned count(T Val) {
- if (!Val)
- return std::numeric_limits<T>::digits;
- if (Val & 0x1)
- return 0;
-
- // Bisection method.
- unsigned ZeroBits = 0;
- T Shift = std::numeric_limits<T>::digits >> 1;
- T Mask = std::numeric_limits<T>::max() >> Shift;
- while (Shift) {
- if ((Val & Mask) == 0) {
- Val >>= Shift;
- ZeroBits |= Shift;
- }
- Shift >>= 1;
- Mask >>= Shift;
- }
- return ZeroBits;
- }
-};
-
-#if defined(__GNUC__) || defined(_MSC_VER)
-template <typename T> struct TrailingZerosCounter<T, 4> {
- static unsigned count(T Val) {
- if (Val == 0)
- return 32;
-
-#if __has_builtin(__builtin_ctz) || defined(__GNUC__)
- return __builtin_ctz(Val);
-#elif defined(_MSC_VER)
- unsigned long Index;
- _BitScanForward(&Index, Val);
- return Index;
-#endif
- }
-};
-
-#if !defined(_MSC_VER) || defined(_M_X64)
-template <typename T> struct TrailingZerosCounter<T, 8> {
- static unsigned count(T Val) {
- if (Val == 0)
- return 64;
-
-#if __has_builtin(__builtin_ctzll) || defined(__GNUC__)
- return __builtin_ctzll(Val);
-#elif defined(_MSC_VER)
- unsigned long Index;
- _BitScanForward64(&Index, Val);
- return Index;
-#endif
- }
-};
-#endif
-#endif
-} // namespace detail
-
/// Count number of 0's from the least significant bit to the most
/// stopping at the first 1.
///
template <typename T> unsigned countTrailingZeros(T Val) {
static_assert(std::is_unsigned_v<T>,
"Only unsigned integral types are allowed.");
- return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val);
-}
-
-namespace detail {
-template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
- static unsigned count(T Val) {
- if (!Val)
- return std::numeric_limits<T>::digits;
-
- // Bisection method.
- unsigned ZeroBits = 0;
- for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
- T Tmp = Val >> Shift;
- if (Tmp)
- Val = Tmp;
- else
- ZeroBits |= Shift;
- }
- return ZeroBits;
- }
-};
-
-#if defined(__GNUC__) || defined(_MSC_VER)
-template <typename T> struct LeadingZerosCounter<T, 4> {
- static unsigned count(T Val) {
- if (Val == 0)
- return 32;
-
-#if __has_builtin(__builtin_clz) || defined(__GNUC__)
- return __builtin_clz(Val);
-#elif defined(_MSC_VER)
- unsigned long Index;
- _BitScanReverse(&Index, Val);
- return Index ^ 31;
-#endif
- }
-};
-
-#if !defined(_MSC_VER) || defined(_M_X64)
-template <typename T> struct LeadingZerosCounter<T, 8> {
- static unsigned count(T Val) {
- if (Val == 0)
- return 64;
-
-#if __has_builtin(__builtin_clzll) || defined(__GNUC__)
- return __builtin_clzll(Val);
-#elif defined(_MSC_VER)
- unsigned long Index;
- _BitScanReverse64(&Index, Val);
- return Index ^ 63;
-#endif
- }
-};
-#endif
-#endif
-} // namespace detail
+ return llvm::countr_zero(Val);
+}
/// Count number of 0's from the most significant bit to the least
/// stopping at the first 1.
template <typename T> unsigned countLeadingZeros(T Val) {
static_assert(std::is_unsigned_v<T>,
"Only unsigned integral types are allowed.");
- return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val);
+ return llvm::countl_zero(Val);
}
/// Get the index of the first set bit starting from the least
template <typename T> unsigned countLeadingOnes(T Value) {
static_assert(std::is_unsigned_v<T>,
"Only unsigned integral types are allowed.");
- return countLeadingZeros<T>(~Value);
+ return llvm::countl_one<T>(Value);
}
/// Count the number of ones from the least significant bit to the first
template <typename T> unsigned countTrailingOnes(T Value) {
static_assert(std::is_unsigned_v<T>,
"Only unsigned integral types are allowed.");
- return countTrailingZeros<T>(~Value);
+ return llvm::countr_one<T>(Value);
}
/// Count the number of set bits in a value.
EXPECT_TRUE(llvm::has_single_bit(static_cast<uint16_t>(kValueS16)));
}
+TEST(BitTest, CountlZero) {
+ uint8_t Z8 = 0;
+ uint16_t Z16 = 0;
+ uint32_t Z32 = 0;
+ uint64_t Z64 = 0;
+ EXPECT_EQ(8, llvm::countl_zero(Z8));
+ EXPECT_EQ(16, llvm::countl_zero(Z16));
+ EXPECT_EQ(32, llvm::countl_zero(Z32));
+ EXPECT_EQ(64, llvm::countl_zero(Z64));
+
+ uint8_t NZ8 = 42;
+ uint16_t NZ16 = 42;
+ uint32_t NZ32 = 42;
+ uint64_t NZ64 = 42;
+ EXPECT_EQ(2, llvm::countl_zero(NZ8));
+ EXPECT_EQ(10, llvm::countl_zero(NZ16));
+ EXPECT_EQ(26, llvm::countl_zero(NZ32));
+ EXPECT_EQ(58, llvm::countl_zero(NZ64));
+
+ EXPECT_EQ(8, llvm::countl_zero(0x00F000FFu));
+ EXPECT_EQ(8, llvm::countl_zero(0x00F12345u));
+ for (unsigned i = 0; i <= 30; ++i) {
+ EXPECT_EQ(int(31 - i), llvm::countl_zero(1u << i));
+ }
+
+ EXPECT_EQ(8, llvm::countl_zero(0x00F1234500F12345ULL));
+ EXPECT_EQ(1, llvm::countl_zero(1ULL << 62));
+ for (unsigned i = 0; i <= 62; ++i) {
+ EXPECT_EQ(int(63 - i), llvm::countl_zero(1ULL << i));
+ }
+}
+
+TEST(BitTest, CountrZero) {
+ uint8_t Z8 = 0;
+ uint16_t Z16 = 0;
+ uint32_t Z32 = 0;
+ uint64_t Z64 = 0;
+ EXPECT_EQ(8, llvm::countr_zero(Z8));
+ EXPECT_EQ(16, llvm::countr_zero(Z16));
+ EXPECT_EQ(32, llvm::countr_zero(Z32));
+ EXPECT_EQ(64, llvm::countr_zero(Z64));
+
+ uint8_t NZ8 = 42;
+ uint16_t NZ16 = 42;
+ uint32_t NZ32 = 42;
+ uint64_t NZ64 = 42;
+ EXPECT_EQ(1, llvm::countr_zero(NZ8));
+ EXPECT_EQ(1, llvm::countr_zero(NZ16));
+ EXPECT_EQ(1, llvm::countr_zero(NZ32));
+ EXPECT_EQ(1, llvm::countr_zero(NZ64));
+}
+
+TEST(BitTest, CountlOne) {
+ for (int i = 30; i >= 0; --i) {
+ // Start with all ones and unset some bit.
+ EXPECT_EQ(31 - i, llvm::countl_one(0xFFFFFFFF ^ (1 << i)));
+ }
+ for (int i = 62; i >= 0; --i) {
+ // Start with all ones and unset some bit.
+ EXPECT_EQ(63 - i, llvm::countl_one(0xFFFFFFFFFFFFFFFFULL ^ (1LL << i)));
+ }
+ for (int i = 30; i >= 0; --i) {
+ // Start with all ones and unset some bit.
+ EXPECT_EQ(31 - i, llvm::countl_one(0xFFFFFFFF ^ (1 << i)));
+ }
+}
+
+TEST(BitTest, CountrOne) {
+ uint8_t AllOnes8 = ~(uint8_t)0;
+ uint16_t AllOnes16 = ~(uint16_t)0;
+ uint32_t AllOnes32 = ~(uint32_t)0;
+ uint64_t AllOnes64 = ~(uint64_t)0;
+ EXPECT_EQ(8, llvm::countr_one(AllOnes8));
+ EXPECT_EQ(16, llvm::countr_one(AllOnes16));
+ EXPECT_EQ(32, llvm::countr_one(AllOnes32));
+ EXPECT_EQ(64, llvm::countr_one(AllOnes64));
+
+ uint8_t X8 = 6;
+ uint16_t X16 = 6;
+ uint32_t X32 = 6;
+ uint64_t X64 = 6;
+ EXPECT_EQ(0, llvm::countr_one(X8));
+ EXPECT_EQ(0, llvm::countr_one(X16));
+ EXPECT_EQ(0, llvm::countr_one(X32));
+ EXPECT_EQ(0, llvm::countr_one(X64));
+
+ uint8_t Y8 = 23;
+ uint16_t Y16 = 23;
+ uint32_t Y32 = 23;
+ uint64_t Y64 = 23;
+ EXPECT_EQ(3, llvm::countr_one(Y8));
+ EXPECT_EQ(3, llvm::countr_one(Y16));
+ EXPECT_EQ(3, llvm::countr_one(Y32));
+ EXPECT_EQ(3, llvm::countr_one(Y64));
+}
+
TEST(BitTest, PopCount) {
EXPECT_EQ(0, llvm::popcount(0U));
EXPECT_EQ(0, llvm::popcount(0ULL));
projects / platform / upstream / llvm.git / commitdiff