[libc] Separate memcpy implementations per arch

As x86_64 implementations is likely to grow up to a point where it's no more manageable to have all implementations in the same file.

diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt
index 371b4010a4dc8074a6fa6bca5cadee7883b8f120..a1cbff0df1e328c87420ea19c2da7a2735633bf4 100644 (file)
--- a/libc/src/string/memory_utils/CMakeLists.txt
+++ b/libc/src/string/memory_utils/CMakeLists.txt
@@ -2,6 +2,7 @@
 add_header_library(
   memory_utils
   HDRS
+    aarch64/memcpy_implementations.h
     bcmp_implementations.h
     bzero_implementations.h
     memcmp_implementations.h
@@ -13,11 +14,12 @@ add_header_library(
     op_generic.h
     op_x86.h
     utils.h
+    x86_64/memcpy_implementations.h
   DEPS
-    libc.src.__support.common
     libc.src.__support.CPP.bit
     libc.src.__support.CPP.cstddef
     libc.src.__support.CPP.type_traits
+    libc.src.__support.macros.config
     libc.src.__support.macros.optimization
 )
 

diff --git a/libc/src/string/memory_utils/aarch64/memcpy_implementations.h b/libc/src/string/memory_utils/aarch64/memcpy_implementations.h
new file mode 100644 (file)
index 0000000..e34eba2
--- /dev/null
+++ b/libc/src/string/memory_utils/aarch64/memcpy_implementations.h
@@ -0,0 +1,48 @@
+//===-- Memcpy implementation for aarch64 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H
+#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H
+
+#include "src/__support/macros/config.h" // LIBC_INLINE
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/utils.h"
+
+#include <stddef.h> // size_t
+
+namespace __llvm_libc {
+
+[[maybe_unused]] LIBC_INLINE void
+inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) {
+  if (count == 0)
+    return;
+  if (count == 1)
+    return builtin::Memcpy<1>::block(dst, src);
+  if (count == 2)
+    return builtin::Memcpy<2>::block(dst, src);
+  if (count == 3)
+    return builtin::Memcpy<3>::block(dst, src);
+  if (count == 4)
+    return builtin::Memcpy<4>::block(dst, src);
+  if (count < 8)
+    return builtin::Memcpy<4>::head_tail(dst, src, count);
+  if (count < 16)
+    return builtin::Memcpy<8>::head_tail(dst, src, count);
+  if (count < 32)
+    return builtin::Memcpy<16>::head_tail(dst, src, count);
+  if (count < 64)
+    return builtin::Memcpy<32>::head_tail(dst, src, count);
+  if (count < 128)
+    return builtin::Memcpy<64>::head_tail(dst, src, count);
+  builtin::Memcpy<16>::block(dst, src);
+  align_to_next_boundary<16, Arg::Src>(dst, src, count);
+  return builtin::Memcpy<64>::loop_and_tail(dst, src, count);
+}
+
+} // namespace __llvm_libc
+
+#endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H

diff --git a/libc/src/string/memory_utils/memcpy_implementations.h b/libc/src/string/memory_utils/memcpy_implementations.h
index 850e0256c7b1d9713d8328c3fc0a6a7a144e9a77..ea5c43bebbbdf48c92e077fd8eddddc47ab62cd2 100644 (file)
--- a/libc/src/string/memory_utils/memcpy_implementations.h
+++ b/libc/src/string/memory_utils/memcpy_implementations.h
@@ -9,17 +9,19 @@
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
 
-#include "src/__support/common.h"
-#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY LIBC_LOOP_NOUNROLL
+#include "src/__support/macros/config.h"       // LIBC_INLINE
+#include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL
 #include "src/__support/macros/properties/architectures.h"
-#include "src/string/memory_utils/op_aarch64.h"
-#include "src/string/memory_utils/op_builtin.h"
-#include "src/string/memory_utils/op_generic.h"
-#include "src/string/memory_utils/op_x86.h"
 #include "src/string/memory_utils/utils.h"
 
 #include <stddef.h> // size_t
 
+#if defined(LIBC_TARGET_ARCH_IS_X86)
+#include "src/string/memory_utils/x86_64/memcpy_implementations.h"
+#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
+#include "src/string/memory_utils/aarch64/memcpy_implementations.h"
+#endif
+
 namespace __llvm_libc {
 
 [[maybe_unused]] LIBC_INLINE void
@@ -30,98 +32,6 @@ inline_memcpy_embedded_tiny(Ptr __restrict dst, CPtr __restrict src,
     builtin::Memcpy<1>::block(dst + offset, src + offset);
 }
 
-#if defined(LIBC_TARGET_ARCH_IS_X86)
-[[maybe_unused]] LIBC_INLINE void
-inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
-  if (count == 0)
-    return;
-  if (count == 1)
-    return builtin::Memcpy<1>::block(dst, src);
-  if (count == 2)
-    return builtin::Memcpy<2>::block(dst, src);
-  if (count == 3)
-    return builtin::Memcpy<3>::block(dst, src);
-  if (count == 4)
-    return builtin::Memcpy<4>::block(dst, src);
-  if (count < 8)
-    return builtin::Memcpy<4>::head_tail(dst, src, count);
-  if (count < 16)
-    return builtin::Memcpy<8>::head_tail(dst, src, count);
-  if (count < 32)
-    return builtin::Memcpy<16>::head_tail(dst, src, count);
-  if (count < 64)
-    return builtin::Memcpy<32>::head_tail(dst, src, count);
-  if (count < 128)
-    return builtin::Memcpy<64>::head_tail(dst, src, count);
-  if (x86::kAvx && count < 256)
-    return builtin::Memcpy<128>::head_tail(dst, src, count);
-  builtin::Memcpy<32>::block(dst, src);
-  align_to_next_boundary<32, Arg::Dst>(dst, src, count);
-  static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32;
-  return builtin::Memcpy<kBlockSize>::loop_and_tail(dst, src, count);
-}
-
-[[maybe_unused]] LIBC_INLINE void
-inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst,
-                                           CPtr __restrict src, size_t count) {
-  // Whether to use rep;movsb exclusively, not at all, or only above a certain
-  // threshold.
-#ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
-#define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1
-#endif
-
-#ifdef LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
-#error LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
-#endif // LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
-
-#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
-#error LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
-#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
-
-  static constexpr size_t kRepMovsbThreshold =
-      LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
-  if constexpr (kRepMovsbThreshold == 0) {
-    return x86::Memcpy::repmovsb(dst, src, count);
-  } else if constexpr (kRepMovsbThreshold == size_t(-1)) {
-    return inline_memcpy_x86(dst, src, count);
-  } else {
-    if (LIBC_UNLIKELY(count >= kRepMovsbThreshold))
-      return x86::Memcpy::repmovsb(dst, src, count);
-    else
-      return inline_memcpy_x86(dst, src, count);
-  }
-}
-#endif // defined(LIBC_TARGET_ARCH_IS_X86)
-
-#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
-[[maybe_unused]] LIBC_INLINE void
-inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) {
-  if (count == 0)
-    return;
-  if (count == 1)
-    return builtin::Memcpy<1>::block(dst, src);
-  if (count == 2)
-    return builtin::Memcpy<2>::block(dst, src);
-  if (count == 3)
-    return builtin::Memcpy<3>::block(dst, src);
-  if (count == 4)
-    return builtin::Memcpy<4>::block(dst, src);
-  if (count < 8)
-    return builtin::Memcpy<4>::head_tail(dst, src, count);
-  if (count < 16)
-    return builtin::Memcpy<8>::head_tail(dst, src, count);
-  if (count < 32)
-    return builtin::Memcpy<16>::head_tail(dst, src, count);
-  if (count < 64)
-    return builtin::Memcpy<32>::head_tail(dst, src, count);
-  if (count < 128)
-    return builtin::Memcpy<64>::head_tail(dst, src, count);
-  builtin::Memcpy<16>::block(dst, src);
-  align_to_next_boundary<16, Arg::Src>(dst, src, count);
-  return builtin::Memcpy<64>::loop_and_tail(dst, src, count);
-}
-#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)
-
 LIBC_INLINE void inline_memcpy(Ptr __restrict dst, CPtr __restrict src,
                                size_t count) {
   using namespace __llvm_libc::builtin;

diff --git a/libc/src/string/memory_utils/x86_64/memcpy_implementations.h b/libc/src/string/memory_utils/x86_64/memcpy_implementations.h
new file mode 100644 (file)
index 0000000..3844e06
--- /dev/null
+++ b/libc/src/string/memory_utils/x86_64/memcpy_implementations.h
@@ -0,0 +1,84 @@
+//===-- Memcpy implementation for x86_64 ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H
+#define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H
+
+#include "src/__support/macros/config.h"       // LIBC_INLINE
+#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/op_x86.h"
+#include "src/string/memory_utils/utils.h"
+
+#include <stddef.h> // size_t
+
+namespace __llvm_libc {
+
+[[maybe_unused]] LIBC_INLINE void
+inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
+  if (count == 0)
+    return;
+  if (count == 1)
+    return builtin::Memcpy<1>::block(dst, src);
+  if (count == 2)
+    return builtin::Memcpy<2>::block(dst, src);
+  if (count == 3)
+    return builtin::Memcpy<3>::block(dst, src);
+  if (count == 4)
+    return builtin::Memcpy<4>::block(dst, src);
+  if (count < 8)
+    return builtin::Memcpy<4>::head_tail(dst, src, count);
+  if (count < 16)
+    return builtin::Memcpy<8>::head_tail(dst, src, count);
+  if (count < 32)
+    return builtin::Memcpy<16>::head_tail(dst, src, count);
+  if (count < 64)
+    return builtin::Memcpy<32>::head_tail(dst, src, count);
+  if (count < 128)
+    return builtin::Memcpy<64>::head_tail(dst, src, count);
+  if (x86::kAvx && count < 256)
+    return builtin::Memcpy<128>::head_tail(dst, src, count);
+  builtin::Memcpy<32>::block(dst, src);
+  align_to_next_boundary<32, Arg::Dst>(dst, src, count);
+  static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32;
+  return builtin::Memcpy<kBlockSize>::loop_and_tail(dst, src, count);
+}
+
+[[maybe_unused]] LIBC_INLINE void
+inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst,
+                                           CPtr __restrict src, size_t count) {
+  // Whether to use rep;movsb exclusively, not at all, or only above a certain
+  // threshold.
+#ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+#define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1
+#endif
+
+#ifdef LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
+#error LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
+#endif // LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
+
+#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+#error LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
+#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+
+  static constexpr size_t kRepMovsbThreshold =
+      LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
+  if constexpr (kRepMovsbThreshold == 0) {
+    return x86::Memcpy::repmovsb(dst, src, count);
+  } else if constexpr (kRepMovsbThreshold == size_t(-1)) {
+    return inline_memcpy_x86(dst, src, count);
+  } else {
+    if (LIBC_UNLIKELY(count >= kRepMovsbThreshold))
+      return x86::Memcpy::repmovsb(dst, src, count);
+    else
+      return inline_memcpy_x86(dst, src, count);
+  }
+}
+
+} // namespace __llvm_libc
+
+#endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index d5ba402e3dc43f58a37a506ee073f97ef98cd9e5..2f0bcf6f41397439cfbbdef9cea123034f96a88d 100644 (file)
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1404,13 +1404,14 @@ libc_support_library(
         "src/string/memory_utils/bzero_implementations.h",
         "src/string/memory_utils/memcmp_implementations.h",
         "src/string/memory_utils/memcpy_implementations.h",
+        "src/string/memory_utils/aarch64/memcpy_implementations.h",
+        "src/string/memory_utils/x86_64/memcpy_implementations.h",
         "src/string/memory_utils/memmove_implementations.h",
         "src/string/memory_utils/memset_implementations.h",
         "src/string/memory_utils/strcmp_implementations.h",
         "src/string/memory_utils/strstr_implementations.h",
     ],
     deps = [
-        ":__support_common",
         ":__support_cpp_array",
         ":__support_cpp_bit",
         ":__support_cpp_cstddef",