1 //===-- Unittests for op_ files -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "memory_check_utils.h"
10 #include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT64
11 #include "src/string/memory_utils/op_aarch64.h"
12 #include "src/string/memory_utils/op_builtin.h"
13 #include "src/string/memory_utils/op_generic.h"
14 #include "src/string/memory_utils/op_riscv.h"
15 #include "src/string/memory_utils/op_x86.h"
16 #include "test/UnitTest/Test.h"
17
18 namespace LIBC_NAMESPACE {
19
20 template <typename T> struct has_head_tail {
21 template <typename C> static char sfinae(decltype(&C::head_tail));
22 template <typename C> static uint16_t sfinae(...);
23 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
24 };
25
26 template <typename T> struct has_loop_and_tail {
27 template <typename C> static char sfinae(decltype(&C::loop_and_tail));
28 template <typename C> static uint16_t sfinae(...);
29 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char);
30 };
31
32 // Allocates two Buffer and extracts two spans out of them, one
33 // aligned and one misaligned. Tests are run on both spans.
34 struct Buffers {
BuffersLIBC_NAMESPACE::Buffers35 Buffers(size_t size)
36 : aligned_buffer(size, Aligned::YES),
37 misaligned_buffer(size, Aligned::NO) {}
38
39 // Returns two spans of 'size' bytes. The first is aligned on
40 // Buffer::kAlign and the second one is unaligned.
spansLIBC_NAMESPACE::Buffers41 cpp::array<cpp::span<char>, 2> spans() {
42 return {aligned_buffer.span(), misaligned_buffer.span()};
43 }
44
45 Buffer aligned_buffer;
46 Buffer misaligned_buffer;
47 };
48
49 using MemcpyImplementations = testing::TypeList<
50 #ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
51 builtin::Memcpy<1>, //
52 builtin::Memcpy<2>, //
53 builtin::Memcpy<3>, //
54 builtin::Memcpy<4>, //
55 builtin::Memcpy<8>, //
56 builtin::Memcpy<16>, //
57 builtin::Memcpy<32>, //
58 builtin::Memcpy<64>
59 #endif // LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
60 >;
61
62 // Convenient helper to turn a span into cpp::byte *.
as_byte(cpp::span<char> span)63 static inline cpp::byte *as_byte(cpp::span<char> span) {
64 return reinterpret_cast<cpp::byte *>(span.data());
65 }
66
67 // Adapt CheckMemcpy signature to op implementation signatures.
68 template <auto FnImpl>
CopyAdaptor(cpp::span<char> dst,cpp::span<char> src,size_t size)69 void CopyAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
70 FnImpl(as_byte(dst), as_byte(src), size);
71 }
72 template <size_t Size, auto FnImpl>
CopyBlockAdaptor(cpp::span<char> dst,cpp::span<char> src,size_t size)73 void CopyBlockAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
74 FnImpl(as_byte(dst), as_byte(src));
75 }
76
TYPED_TEST(LlvmLibcOpTest,Memcpy,MemcpyImplementations)77 TYPED_TEST(LlvmLibcOpTest, Memcpy, MemcpyImplementations) {
78 using Impl = ParamType;
79 constexpr size_t kSize = Impl::SIZE;
80 { // Test block operation
81 static constexpr auto BlockImpl = CopyBlockAdaptor<kSize, Impl::block>;
82 Buffers SrcBuffer(kSize);
83 Buffers DstBuffer(kSize);
84 for (auto src : SrcBuffer.spans()) {
85 Randomize(src);
86 for (auto dst : DstBuffer.spans()) {
87 ASSERT_TRUE(CheckMemcpy<BlockImpl>(dst, src, kSize));
88 }
89 }
90 }
91 { // Test head tail operations from kSize to 2 * kSize.
92 static constexpr auto HeadTailImpl = CopyAdaptor<Impl::head_tail>;
93 Buffer SrcBuffer(2 * kSize);
94 Buffer DstBuffer(2 * kSize);
95 Randomize(SrcBuffer.span());
96 for (size_t size = kSize; size < 2 * kSize; ++size) {
97 auto src = SrcBuffer.span().subspan(0, size);
98 auto dst = DstBuffer.span().subspan(0, size);
99 ASSERT_TRUE(CheckMemcpy<HeadTailImpl>(dst, src, size));
100 }
101 }
102 { // Test loop operations from kSize to 3 * kSize.
103 if constexpr (kSize > 1) {
104 static constexpr auto LoopImpl = CopyAdaptor<Impl::loop_and_tail>;
105 Buffer SrcBuffer(3 * kSize);
106 Buffer DstBuffer(3 * kSize);
107 Randomize(SrcBuffer.span());
108 for (size_t size = kSize; size < 3 * kSize; ++size) {
109 auto src = SrcBuffer.span().subspan(0, size);
110 auto dst = DstBuffer.span().subspan(0, size);
111 ASSERT_TRUE(CheckMemcpy<LoopImpl>(dst, src, size));
112 }
113 }
114 }
115 }
116
117 using MemsetImplementations = testing::TypeList<
118 #ifdef LLVM_LIBC_HAS_BUILTIN_MEMSET_INLINE
119 builtin::Memset<1>, //
120 builtin::Memset<2>, //
121 builtin::Memset<3>, //
122 builtin::Memset<4>, //
123 builtin::Memset<8>, //
124 builtin::Memset<16>, //
125 builtin::Memset<32>, //
126 builtin::Memset<64>,
127 #endif
128 #ifdef LIBC_TYPES_HAS_INT64
129 generic::Memset<uint64_t>, generic::Memset<cpp::array<uint64_t, 2>>,
130 #endif // LIBC_TYPES_HAS_INT64
131 #ifdef __AVX512F__
132 generic::Memset<generic_v512>, generic::Memset<cpp::array<generic_v512, 2>>,
133 #endif
134 #ifdef __AVX__
135 generic::Memset<generic_v256>, generic::Memset<cpp::array<generic_v256, 2>>,
136 #endif
137 #ifdef __SSE2__
138 generic::Memset<generic_v128>, generic::Memset<cpp::array<generic_v128, 2>>,
139 #endif
140 generic::Memset<uint32_t>, generic::Memset<cpp::array<uint32_t, 2>>, //
141 generic::Memset<uint16_t>, generic::Memset<cpp::array<uint16_t, 2>>, //
142 generic::Memset<uint8_t>, generic::Memset<cpp::array<uint8_t, 2>>, //
143 generic::MemsetSequence<uint8_t, uint8_t>, //
144 generic::MemsetSequence<uint16_t, uint8_t>, //
145 generic::MemsetSequence<uint32_t, uint16_t, uint8_t> //
146 >;
147
148 // Adapt CheckMemset signature to op implementation signatures.
149 template <auto FnImpl>
SetAdaptor(cpp::span<char> dst,uint8_t value,size_t size)150 void SetAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
151 FnImpl(as_byte(dst), value, size);
152 }
153 template <size_t Size, auto FnImpl>
SetBlockAdaptor(cpp::span<char> dst,uint8_t value,size_t size)154 void SetBlockAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
155 FnImpl(as_byte(dst), value);
156 }
157
TYPED_TEST(LlvmLibcOpTest,Memset,MemsetImplementations)158 TYPED_TEST(LlvmLibcOpTest, Memset, MemsetImplementations) {
159 using Impl = ParamType;
160 constexpr size_t kSize = Impl::SIZE;
161 { // Test block operation
162 static constexpr auto BlockImpl = SetBlockAdaptor<kSize, Impl::block>;
163 Buffers DstBuffer(kSize);
164 for (uint8_t value : cpp::array<uint8_t, 3>{0, 1, 255}) {
165 for (auto dst : DstBuffer.spans()) {
166 ASSERT_TRUE(CheckMemset<BlockImpl>(dst, value, kSize));
167 }
168 }
169 }
170 if constexpr (has_head_tail<Impl>::value) {
171 // Test head tail operations from kSize to 2 * kSize.
172 static constexpr auto HeadTailImpl = SetAdaptor<Impl::head_tail>;
173 Buffer DstBuffer(2 * kSize);
174 for (size_t size = kSize; size < 2 * kSize; ++size) {
175 const char value = size % 10;
176 auto dst = DstBuffer.span().subspan(0, size);
177 ASSERT_TRUE(CheckMemset<HeadTailImpl>(dst, value, size));
178 }
179 }
180 if constexpr (has_loop_and_tail<Impl>::value) {
181 // Test loop operations from kSize to 3 * kSize.
182 if constexpr (kSize > 1) {
183 static constexpr auto LoopImpl = SetAdaptor<Impl::loop_and_tail>;
184 Buffer DstBuffer(3 * kSize);
185 for (size_t size = kSize; size < 3 * kSize; ++size) {
186 const char value = size % 10;
187 auto dst = DstBuffer.span().subspan(0, size);
188 ASSERT_TRUE((CheckMemset<LoopImpl>(dst, value, size)));
189 }
190 }
191 }
192 }
193
194 using BcmpImplementations = testing::TypeList<
195 #ifdef LIBC_TARGET_ARCH_IS_X86_64
196 #ifdef __SSE4_1__
197 generic::Bcmp<__m128i>,
198 #endif // __SSE4_1__
199 #ifdef __AVX2__
200 generic::Bcmp<__m256i>,
201 #endif // __AVX2__
202 #ifdef __AVX512BW__
203 generic::Bcmp<__m512i>,
204 #endif // __AVX512BW__
205
206 #endif // LIBC_TARGET_ARCH_IS_X86_64
207 #ifdef LIBC_TARGET_ARCH_IS_AARCH64
208 aarch64::Bcmp<16>, //
209 aarch64::Bcmp<32>,
210 #endif
211 #ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
212 generic::Bcmp<uint16_t>,
213 generic::Bcmp<uint32_t>, //
214 #ifdef LIBC_TYPES_HAS_INT64
215 generic::Bcmp<uint64_t>,
216 #endif // LIBC_TYPES_HAS_INT64
217 generic::BcmpSequence<uint16_t, uint8_t>,
218 generic::BcmpSequence<uint32_t, uint8_t>, //
219 generic::BcmpSequence<uint32_t, uint16_t>, //
220 generic::BcmpSequence<uint32_t, uint16_t, uint8_t>,
221 #endif // LIBC_TARGET_ARCH_IS_ARM
222 generic::BcmpSequence<uint8_t, uint8_t>,
223 generic::BcmpSequence<uint8_t, uint8_t, uint8_t>, //
224 generic::Bcmp<uint8_t>>;
225
226 // Adapt CheckBcmp signature to op implementation signatures.
227 template <auto FnImpl>
CmpAdaptor(cpp::span<char> p1,cpp::span<char> p2,size_t size)228 int CmpAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
229 return (int)FnImpl(as_byte(p1), as_byte(p2), size);
230 }
231 template <size_t Size, auto FnImpl>
CmpBlockAdaptor(cpp::span<char> p1,cpp::span<char> p2,size_t size)232 int CmpBlockAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
233 return (int)FnImpl(as_byte(p1), as_byte(p2));
234 }
235
TYPED_TEST(LlvmLibcOpTest,Bcmp,BcmpImplementations)236 TYPED_TEST(LlvmLibcOpTest, Bcmp, BcmpImplementations) {
237 using Impl = ParamType;
238 constexpr size_t kSize = Impl::SIZE;
239 { // Test block operation
240 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
241 Buffers Buffer1(kSize);
242 Buffers Buffer2(kSize);
243 for (auto span1 : Buffer1.spans()) {
244 Randomize(span1);
245 for (auto span2 : Buffer2.spans())
246 ASSERT_TRUE((CheckBcmp<BlockImpl>(span1, span2, kSize)));
247 }
248 }
249 if constexpr (has_head_tail<Impl>::value) {
250 // Test head tail operations from kSize to 2 * kSize.
251 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
252 Buffer Buffer1(2 * kSize);
253 Buffer Buffer2(2 * kSize);
254 Randomize(Buffer1.span());
255 for (size_t size = kSize; size < 2 * kSize; ++size) {
256 auto span1 = Buffer1.span().subspan(0, size);
257 auto span2 = Buffer2.span().subspan(0, size);
258 ASSERT_TRUE((CheckBcmp<HeadTailImpl>(span1, span2, size)));
259 }
260 }
261 if constexpr (has_loop_and_tail<Impl>::value) {
262 // Test loop operations from kSize to 3 * kSize.
263 if constexpr (kSize > 1) {
264 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
265 Buffer Buffer1(3 * kSize);
266 Buffer Buffer2(3 * kSize);
267 Randomize(Buffer1.span());
268 for (size_t size = kSize; size < 3 * kSize; ++size) {
269 auto span1 = Buffer1.span().subspan(0, size);
270 auto span2 = Buffer2.span().subspan(0, size);
271 ASSERT_TRUE((CheckBcmp<LoopImpl>(span1, span2, size)));
272 }
273 }
274 }
275 }
276
277 using MemcmpImplementations = testing::TypeList<
278 #ifdef LIBC_TARGET_ARCH_IS_X86_64
279 #ifdef __SSE2__
280 generic::Memcmp<__m128i>, //
281 #endif
282 #ifdef __AVX2__
283 generic::Memcmp<__m256i>, //
284 #endif
285 #ifdef __AVX512BW__
286 generic::Memcmp<__m512i>, //
287 #endif
288 #endif // LIBC_TARGET_ARCH_IS_X86_64
289 #ifdef LIBC_TARGET_ARCH_IS_AARCH64
290 generic::Memcmp<uint8x16_t>, //
291 generic::Memcmp<uint8x16x2_t>,
292 #endif
293 #ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM
294 generic::Memcmp<uint16_t>,
295 generic::Memcmp<uint32_t>, //
296 #ifdef LIBC_TYPES_HAS_INT64
297 generic::Memcmp<uint64_t>,
298 #endif // LIBC_TYPES_HAS_INT64
299 generic::MemcmpSequence<uint16_t, uint8_t>,
300 generic::MemcmpSequence<uint32_t, uint16_t, uint8_t>, //
301 #endif // LIBC_TARGET_ARCH_IS_ARM
302 generic::MemcmpSequence<uint8_t, uint8_t>,
303 generic::MemcmpSequence<uint8_t, uint8_t, uint8_t>,
304 generic::Memcmp<uint8_t>>;
305
TYPED_TEST(LlvmLibcOpTest,Memcmp,MemcmpImplementations)306 TYPED_TEST(LlvmLibcOpTest, Memcmp, MemcmpImplementations) {
307 using Impl = ParamType;
308 constexpr size_t kSize = Impl::SIZE;
309 { // Test block operation
310 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>;
311 Buffers Buffer1(kSize);
312 Buffers Buffer2(kSize);
313 for (auto span1 : Buffer1.spans()) {
314 Randomize(span1);
315 for (auto span2 : Buffer2.spans())
316 ASSERT_TRUE((CheckMemcmp<BlockImpl>(span1, span2, kSize)));
317 }
318 }
319 if constexpr (has_head_tail<Impl>::value) {
320 // Test head tail operations from kSize to 2 * kSize.
321 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>;
322 Buffer Buffer1(2 * kSize);
323 Buffer Buffer2(2 * kSize);
324 Randomize(Buffer1.span());
325 for (size_t size = kSize; size < 2 * kSize; ++size) {
326 auto span1 = Buffer1.span().subspan(0, size);
327 auto span2 = Buffer2.span().subspan(0, size);
328 ASSERT_TRUE((CheckMemcmp<HeadTailImpl>(span1, span2, size)));
329 }
330 }
331 if constexpr (has_loop_and_tail<Impl>::value) {
332 // Test loop operations from kSize to 3 * kSize.
333 if constexpr (kSize > 1) {
334 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>;
335 Buffer Buffer1(3 * kSize);
336 Buffer Buffer2(3 * kSize);
337 Randomize(Buffer1.span());
338 for (size_t size = kSize; size < 3 * kSize; ++size) {
339 auto span1 = Buffer1.span().subspan(0, size);
340 auto span2 = Buffer2.span().subspan(0, size);
341 ASSERT_TRUE((CheckMemcmp<LoopImpl>(span1, span2, size)));
342 }
343 }
344 }
345 }
346
347 } // namespace LIBC_NAMESPACE
348