From 060a43ced27ead1ebbe2b0421599f1a03bff25e5 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 29 Sep 2022 13:51:20 +0000 Subject: [PATCH] [libc][NFC] Move alignment utils to utils.h --- libc/src/string/memory_utils/elements.h | 7 +- libc/src/string/memory_utils/utils.h | 84 ++++++++++++++++++++++++ libc/test/src/string/memory_utils/utils_test.cpp | 67 +++++++++++++++++++ 3 files changed, 152 insertions(+), 6 deletions(-) diff --git a/libc/src/string/memory_utils/elements.h b/libc/src/string/memory_utils/elements.h index a5aa30d..f5a3830 100644 --- a/libc/src/string/memory_utils/elements.h +++ b/libc/src/string/memory_utils/elements.h @@ -423,8 +423,6 @@ template struct Loop { } }; -enum class Arg { _1, _2, Dst = _1, Src = _2, Lhs = _1, Rhs = _2 }; - namespace internal { template struct ArgSelector {}; @@ -454,10 +452,7 @@ template struct Align { int additional_bumps = 0) { auto &aligned_ptr = ArgSelector::Select(p1ref, p2ref); auto offset = offset_to_next_aligned(aligned_ptr); - offset += additional_bumps * Alignment; - p1ref += offset; - p2ref += offset; - size -= offset; + adjust(offset + additional_bumps * Alignment, p1ref, p2ref, size); aligned_ptr = assume_aligned(aligned_ptr); } }; diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h index 7197a7e..d915835 100644 --- a/libc/src/string/memory_utils/utils.h +++ b/libc/src/string/memory_utils/utils.h @@ -28,6 +28,13 @@ namespace __llvm_libc { +// Allows compile time error reporting in `if constexpr` branches. +template +static void deferred_static_assert(const char *msg) { + static_assert(flag, "compilation error"); + (void)msg; +} + // Return whether `value` is zero or a power of two. static constexpr bool is_power2_or_zero(size_t value) { return (value & (value - 1U)) == 0; @@ -75,6 +82,83 @@ static inline intptr_t offset_to_next_cache_line(const void *ptr) { template static T *assume_aligned(T *ptr) { return reinterpret_cast(__builtin_assume_aligned(ptr, alignment)); } +#if defined __has_builtin +#if __has_builtin(__builtin_memcpy_inline) +#define LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE +#endif +#endif + +// Performs a constant count copy. +template +static inline void memcpy_inline(void *__restrict dst, + const void *__restrict src) { +#ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE + __builtin_memcpy_inline(dst, src, Size); +#else + for (size_t i = 0; i < Size; ++i) + static_cast(dst)[i] = static_cast(src)[i]; +#endif +} + +using Ptr = char *; // Pointer to raw data. +using CPtr = const char *; // Const pointer to raw data. + +// Loads bytes from memory (possibly unaligned) and materializes them as type. +template static inline T load(CPtr ptr) { + T Out; + memcpy_inline(&Out, ptr); + return Out; +} + +// Stores a value of type T in memory (possibly unaligned) +template static inline void store(Ptr ptr, T value) { + memcpy_inline(ptr, &value); +} + +// For an operation like memset that operates on a pointer and a count, advances +// the pointer by offset bytes and decrease count by the same amount. +static inline void adjust(ptrdiff_t offset, Ptr &ptr, size_t &count) { + ptr += offset; + count -= offset; +} + +// For an operation like memcpy or memcmp that operates on two pointers and a +// count, advances the pointers by offset bytes and decrease count by the same +// amount. +template +static inline void adjust(ptrdiff_t offset, T1 *__restrict &p1, + T2 *__restrict &p2, size_t &count) { + p1 += offset; + p2 += offset; + count -= offset; +} + +// For an operation like memset that operates on a pointer and a count, advances +// the pointer so it is aligned to SIZE bytes and decrease count by the same +// amount. +// We make sure the compiler knows about the adjusted pointer alignment. +template void align(Ptr &ptr, size_t &count) { + adjust(offset_to_next_aligned(ptr), ptr, count); + ptr = assume_aligned(ptr); +} + +// For an operation like memcpy or memcmp that operates on two pointers and a +// count, advances the pointers so one of them gets aligned to SIZE bytes and +// decrease count by the same amount. +// We make sure the compiler knows about the adjusted pointer alignment. +enum class Arg { _1, _2, Dst = _1, Src = _2, Lhs = _1, Rhs = _2 }; +template +void align(T1 *__restrict &p1, T2 *__restrict &p2, size_t &count) { + if constexpr (AlignOn == Arg::_1) { + adjust(offset_to_next_aligned(p1), p1, p2, count); + p1 = assume_aligned(p1); + } else if constexpr (AlignOn == Arg::_2) { + adjust(offset_to_next_aligned(p2), p1, p2, count); + p2 = assume_aligned(p2); + } else { + deferred_static_assert("AlignOn must be either Arg::_1 or Arg::_2"); + } +} } // namespace __llvm_libc diff --git a/libc/test/src/string/memory_utils/utils_test.cpp b/libc/test/src/string/memory_utils/utils_test.cpp index f76d3a7..a20c090 100644 --- a/libc/test/src/string/memory_utils/utils_test.cpp +++ b/libc/test/src/string/memory_utils/utils_test.cpp @@ -104,4 +104,71 @@ TEST(LlvmLibcUtilsTest, OffsetToNextCacheLine) { EXPECT_EQ(offset_to_next_cache_line(forge(LLVM_LIBC_CACHELINE_SIZE - 1)), I(1)); } + +TEST(LlvmLibcUtilsTest, Adjust1) { + char a; + const size_t base_size = 10; + for (size_t I = -2; I < 2; ++I) { + auto *ptr = &a; + size_t size = base_size; + adjust(I, ptr, size); + EXPECT_EQ(intptr_t(ptr), intptr_t(&a + I)); + EXPECT_EQ(size, base_size - I); + } +} + +TEST(LlvmLibcUtilsTest, Adjust2) { + char a, b; + const size_t base_size = 10; + for (size_t I = -2; I < 2; ++I) { + auto *p1 = &a; + auto *p2 = &b; + size_t size = base_size; + adjust(I, p1, p2, size); + EXPECT_EQ(intptr_t(p1), intptr_t(&a + I)); + EXPECT_EQ(intptr_t(p2), intptr_t(&b + I)); + EXPECT_EQ(size, base_size - I); + } +} + +TEST(LlvmLibcUtilsTest, Align1) { + char a; + const size_t base_size = 10; + { + auto *ptr = &a; + size_t size = base_size; + align<128>(ptr, size); + EXPECT_TRUE(uintptr_t(ptr) % 128 == 0); + EXPECT_GE(ptr, &a); + EXPECT_EQ(size_t(ptr - &a), base_size - size); + } +} + +TEST(LlvmLibcUtilsTest, Align2) { + char a, b; + const size_t base_size = 10; + { + auto *p1 = &a; + auto *p2 = &b; + size_t size = base_size; + align<128, Arg::_1>(p1, p2, size); + EXPECT_TRUE(uintptr_t(p1) % 128 == 0); + EXPECT_GE(p1, &a); + EXPECT_GE(p2, &b); + EXPECT_EQ(size_t(p1 - &a), base_size - size); + EXPECT_EQ(size_t(p2 - &b), base_size - size); + } + { + auto *p1 = &a; + auto *p2 = &b; + size_t size = base_size; + align<128, Arg::_2>(p1, p2, size); + EXPECT_TRUE(uintptr_t(p2) % 128 == 0); + EXPECT_GE(p1, &a); + EXPECT_GE(p2, &b); + EXPECT_EQ(size_t(p1 - &a), base_size - size); + EXPECT_EQ(size_t(p2 - &b), base_size - size); + } +} + } // namespace __llvm_libc -- 2.7.4