From a5f4f12b5e8f461cec263d61fda0bca83f290605 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Tue, 25 Oct 2022 14:36:06 +0000 Subject: [PATCH] [libc][NFC] move memmove implementation Moving memmove implementation to its own file for symmetry with other mem functions. Differential Revision: https://reviews.llvm.org/D136687 --- libc/src/string/CMakeLists.txt | 2 +- libc/src/string/memmove.cpp | 104 +-------------- libc/src/string/memory_utils/CMakeLists.txt | 9 ++ .../memory_utils/memmove_implementations.h | 118 ++++++++++++++++++ .../llvm-project-overlay/libc/BUILD.bazel | 2 +- 5 files changed, 132 insertions(+), 103 deletions(-) create mode 100644 libc/src/string/memory_utils/memmove_implementations.h diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt index 624afb4d470a..e300a036655a 100644 --- a/libc/src/string/CMakeLists.txt +++ b/libc/src/string/CMakeLists.txt @@ -461,7 +461,7 @@ function(add_memmove memmove_name) SRCS ${LIBC_SOURCE_DIR}/src/string/memmove.cpp HDRS ${LIBC_SOURCE_DIR}/src/string/memmove.h DEPENDS - .memory_utils.memory_utils + .memory_utils.memcpy_implementation libc.include.string ${ARGN} ) diff --git a/libc/src/string/memmove.cpp b/libc/src/string/memmove.cpp index a42ced3fc36b..ba1b29bf9876 100644 --- a/libc/src/string/memmove.cpp +++ b/libc/src/string/memmove.cpp @@ -7,112 +7,14 @@ //===----------------------------------------------------------------------===// #include "src/string/memmove.h" - -#include "src/__support/common.h" -#include "src/string/memory_utils/op_aarch64.h" -#include "src/string/memory_utils/op_builtin.h" -#include "src/string/memory_utils/op_generic.h" -#include "src/string/memory_utils/op_x86.h" -#include // size_t, ptrdiff_t - -#include +#include "src/string/memory_utils/memmove_implementations.h" +#include // size_t namespace __llvm_libc { -[[maybe_unused]] static inline void -inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) { - if ((count == 0) || (dst == src)) - return; - if (dst < src) { -#pragma nounroll - for (size_t offset = 0; offset < count; ++offset) - builtin::Memcpy<1>::block(dst + offset, src + offset); - } else { -#pragma nounroll - for (ptrdiff_t offset = count - 1; offset >= 0; --offset) - builtin::Memcpy<1>::block(dst + offset, src + offset); - } -} - -template -[[maybe_unused]] static inline void inline_memmove_generic(Ptr dst, CPtr src, - size_t count) { - if (count == 0) - return; - if (count == 1) - return generic::Memmove<1, MaxSize>::block(dst, src); - if (count <= 4) - return generic::Memmove<2, MaxSize>::head_tail(dst, src, count); - if (count <= 8) - return generic::Memmove<4, MaxSize>::head_tail(dst, src, count); - if (count <= 16) - return generic::Memmove<8, MaxSize>::head_tail(dst, src, count); - if (count <= 32) - return generic::Memmove<16, MaxSize>::head_tail(dst, src, count); - if (count <= 64) - return generic::Memmove<32, MaxSize>::head_tail(dst, src, count); - if (count <= 128) - return generic::Memmove<64, MaxSize>::head_tail(dst, src, count); - if (dst < src) { - generic::Memmove<32, MaxSize>::template align_forward(dst, src, - count); - return generic::Memmove<64, MaxSize>::loop_and_tail_forward(dst, src, - count); - } else { - generic::Memmove<32, MaxSize>::template align_backward(dst, src, - count); - return generic::Memmove<64, MaxSize>::loop_and_tail_backward(dst, src, - count); - } -} - -static inline void inline_memmove(Ptr dst, CPtr src, size_t count) { -#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) -#if defined(LLVM_LIBC_ARCH_X86) - static constexpr size_t kMaxSize = x86::kAvx512F ? 64 - : x86::kAvx ? 32 - : x86::kSse2 ? 16 - : 8; -#elif defined(LLVM_LIBC_ARCH_AARCH64) - static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8; -#endif - // return inline_memmove_generic(dst, src, count); - if (count == 0) - return; - if (count == 1) - return generic::Memmove<1, kMaxSize>::block(dst, src); - if (count <= 4) - return generic::Memmove<2, kMaxSize>::head_tail(dst, src, count); - if (count <= 8) - return generic::Memmove<4, kMaxSize>::head_tail(dst, src, count); - if (count <= 16) - return generic::Memmove<8, kMaxSize>::head_tail(dst, src, count); - if (count <= 32) - return generic::Memmove<16, kMaxSize>::head_tail(dst, src, count); - if (count <= 64) - return generic::Memmove<32, kMaxSize>::head_tail(dst, src, count); - if (count <= 128) - return generic::Memmove<64, kMaxSize>::head_tail(dst, src, count); - if (dst < src) { - generic::Memmove<32, kMaxSize>::align_forward(dst, src, count); - return generic::Memmove<64, kMaxSize>::loop_and_tail_forward(dst, src, - count); - } else { - generic::Memmove<32, kMaxSize>::align_backward(dst, src, count); - return generic::Memmove<64, kMaxSize>::loop_and_tail_backward(dst, src, - count); - } -#elif defined(LLVM_LIBC_ARCH_ARM) - return inline_memmove_embedded_tiny(dst, src, count); -#else -#error "Unsupported platform" -#endif -} - LLVM_LIBC_FUNCTION(void *, memmove, (void *dst, const void *src, size_t count)) { - inline_memmove(reinterpret_cast(dst), reinterpret_cast(src), - count); + inline_memmove(dst, src, count); return dst; } diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt index bee8c39a4a80..1436f7c677cf 100644 --- a/libc/src/string/memory_utils/CMakeLists.txt +++ b/libc/src/string/memory_utils/CMakeLists.txt @@ -6,6 +6,7 @@ add_header_library( bzero_implementations.h memcmp_implementations.h memcpy_implementations.h + memmove_implementations.h memset_implementations.h op_aarch64.h op_builtin.h @@ -27,6 +28,14 @@ add_header_library( .memory_utils ) +add_header_library( + memmove_implementation + HDRS + memmove_implementations.h + DEPS + .memory_utils +) + add_header_library( memcmp_implementation HDRS diff --git a/libc/src/string/memory_utils/memmove_implementations.h b/libc/src/string/memory_utils/memmove_implementations.h new file mode 100644 index 000000000000..91f77a67bf2b --- /dev/null +++ b/libc/src/string/memory_utils/memmove_implementations.h @@ -0,0 +1,118 @@ +//===-- Memmove implementation ----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H +#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H + +#include "src/__support/common.h" +#include "src/string/memory_utils/op_aarch64.h" +#include "src/string/memory_utils/op_builtin.h" +#include "src/string/memory_utils/op_generic.h" +#include "src/string/memory_utils/op_x86.h" +#include // size_t, ptrdiff_t + +namespace __llvm_libc { + +[[maybe_unused]] static inline void +inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) { + if ((count == 0) || (dst == src)) + return; + if (dst < src) { +#pragma nounroll + for (size_t offset = 0; offset < count; ++offset) + builtin::Memcpy<1>::block(dst + offset, src + offset); + } else { +#pragma nounroll + for (ptrdiff_t offset = count - 1; offset >= 0; --offset) + builtin::Memcpy<1>::block(dst + offset, src + offset); + } +} + +template +[[maybe_unused]] static inline void inline_memmove_generic(Ptr dst, CPtr src, + size_t count) { + if (count == 0) + return; + if (count == 1) + return generic::Memmove<1, MaxSize>::block(dst, src); + if (count <= 4) + return generic::Memmove<2, MaxSize>::head_tail(dst, src, count); + if (count <= 8) + return generic::Memmove<4, MaxSize>::head_tail(dst, src, count); + if (count <= 16) + return generic::Memmove<8, MaxSize>::head_tail(dst, src, count); + if (count <= 32) + return generic::Memmove<16, MaxSize>::head_tail(dst, src, count); + if (count <= 64) + return generic::Memmove<32, MaxSize>::head_tail(dst, src, count); + if (count <= 128) + return generic::Memmove<64, MaxSize>::head_tail(dst, src, count); + if (dst < src) { + generic::Memmove<32, MaxSize>::template align_forward(dst, src, + count); + return generic::Memmove<64, MaxSize>::loop_and_tail_forward(dst, src, + count); + } else { + generic::Memmove<32, MaxSize>::template align_backward(dst, src, + count); + return generic::Memmove<64, MaxSize>::loop_and_tail_backward(dst, src, + count); + } +} + +static inline void inline_memmove(Ptr dst, CPtr src, size_t count) { +#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) +#if defined(LLVM_LIBC_ARCH_X86) + static constexpr size_t kMaxSize = x86::kAvx512F ? 64 + : x86::kAvx ? 32 + : x86::kSse2 ? 16 + : 8; +#elif defined(LLVM_LIBC_ARCH_AARCH64) + static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8; +#endif + // return inline_memmove_generic(dst, src, count); + if (count == 0) + return; + if (count == 1) + return generic::Memmove<1, kMaxSize>::block(dst, src); + if (count <= 4) + return generic::Memmove<2, kMaxSize>::head_tail(dst, src, count); + if (count <= 8) + return generic::Memmove<4, kMaxSize>::head_tail(dst, src, count); + if (count <= 16) + return generic::Memmove<8, kMaxSize>::head_tail(dst, src, count); + if (count <= 32) + return generic::Memmove<16, kMaxSize>::head_tail(dst, src, count); + if (count <= 64) + return generic::Memmove<32, kMaxSize>::head_tail(dst, src, count); + if (count <= 128) + return generic::Memmove<64, kMaxSize>::head_tail(dst, src, count); + if (dst < src) { + generic::Memmove<32, kMaxSize>::align_forward(dst, src, count); + return generic::Memmove<64, kMaxSize>::loop_and_tail_forward(dst, src, + count); + } else { + generic::Memmove<32, kMaxSize>::align_backward(dst, src, count); + return generic::Memmove<64, kMaxSize>::loop_and_tail_backward(dst, src, + count); + } +#elif defined(LLVM_LIBC_ARCH_ARM) + return inline_memmove_embedded_tiny(dst, src, count); +#else +#error "Unsupported platform" +#endif +} + +static inline void inline_memmove(void *dst, const void *src, size_t count) { + inline_memmove(reinterpret_cast(dst), reinterpret_cast(src), + count); +} + +} // namespace __llvm_libc + +#endif /* LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H */ diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 27b56b92e591..0d1aef2e7536 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -1019,6 +1019,7 @@ cc_library( "src/string/memory_utils/bzero_implementations.h", "src/string/memory_utils/memcmp_implementations.h", "src/string/memory_utils/memcpy_implementations.h", + "src/string/memory_utils/memmove_implementations.h", "src/string/memory_utils/memset_implementations.h", ], deps = [ @@ -1082,7 +1083,6 @@ libc_function( features = no_sanitize_features, deps = [ ":__support_common", - ":__support_integer_operations", ":string_memory_utils", ], )