1 // Copyright 2018 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef V8_UTILS_MEMCOPY_H_
6 #define V8_UTILS_MEMCOPY_H_
7
8 #include <stdint.h>
9 #include <stdlib.h>
10 #include <string.h>
11
12 #include <algorithm>
13
14 #include "src/base/logging.h"
15 #include "src/base/macros.h"
16 #include "src/base/platform/wrappers.h"
17
18 namespace v8 {
19 namespace internal {
20
21 using Address = uintptr_t;
22
23 // ----------------------------------------------------------------------------
24 // Generated memcpy/memmove for ia32, arm, and mips.
25
26 void init_memcopy_functions();
27
28 #if defined(V8_TARGET_ARCH_IA32)
29 // Limit below which the extra overhead of the MemCopy function is likely
30 // to outweigh the benefits of faster copying.
31 const size_t kMinComplexMemCopy = 64;
32
33 // Copy memory area. No restrictions.
34 V8_EXPORT_PRIVATE void MemMove(void* dest, const void* src, size_t size);
35 using MemMoveFunction = void (*)(void* dest, const void* src, size_t size);
36
37 // Keep the distinction of "move" vs. "copy" for the benefit of other
38 // architectures.
MemCopy(void * dest,const void * src,size_t size)39 V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
40 MemMove(dest, src, size);
41 }
42 #elif defined(V8_HOST_ARCH_ARM)
43 using MemCopyUint8Function = void (*)(uint8_t* dest, const uint8_t* src,
44 size_t size);
45 V8_EXPORT_PRIVATE extern MemCopyUint8Function memcopy_uint8_function;
MemCopyUint8Wrapper(uint8_t * dest,const uint8_t * src,size_t chars)46 V8_INLINE void MemCopyUint8Wrapper(uint8_t* dest, const uint8_t* src,
47 size_t chars) {
48 memcpy(dest, src, chars);
49 }
50 // For values < 16, the assembler function is slower than the inlined C code.
51 const size_t kMinComplexMemCopy = 16;
MemCopy(void * dest,const void * src,size_t size)52 V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
53 (*memcopy_uint8_function)(reinterpret_cast<uint8_t*>(dest),
54 reinterpret_cast<const uint8_t*>(src), size);
55 }
MemMove(void * dest,const void * src,size_t size)56 V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
57 size_t size) {
58 memmove(dest, src, size);
59 }
60
61 // For values < 12, the assembler function is slower than the inlined C code.
62 const int kMinComplexConvertMemCopy = 12;
63 #elif defined(V8_HOST_ARCH_MIPS)
64 using MemCopyUint8Function = void (*)(uint8_t* dest, const uint8_t* src,
65 size_t size);
66 V8_EXPORT_PRIVATE extern MemCopyUint8Function memcopy_uint8_function;
MemCopyUint8Wrapper(uint8_t * dest,const uint8_t * src,size_t chars)67 V8_INLINE void MemCopyUint8Wrapper(uint8_t* dest, const uint8_t* src,
68 size_t chars) {
69 memcpy(dest, src, chars);
70 }
71 // For values < 16, the assembler function is slower than the inlined C code.
72 const size_t kMinComplexMemCopy = 16;
MemCopy(void * dest,const void * src,size_t size)73 V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
74 (*memcopy_uint8_function)(reinterpret_cast<uint8_t*>(dest),
75 reinterpret_cast<const uint8_t*>(src), size);
76 }
MemMove(void * dest,const void * src,size_t size)77 V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
78 size_t size) {
79 memmove(dest, src, size);
80 }
81 #else
82 // Copy memory area to disjoint memory area.
MemCopy(void * dest,const void * src,size_t size)83 inline void MemCopy(void* dest, const void* src, size_t size) {
84 // Fast path for small sizes. The compiler will expand the {memcpy} for small
85 // fixed sizes to a sequence of move instructions. This avoids the overhead of
86 // the general {memcpy} function.
87 switch (size) {
88 #define CASE(N) \
89 case N: \
90 memcpy(dest, src, N); \
91 return;
92 CASE(1)
93 CASE(2)
94 CASE(3)
95 CASE(4)
96 CASE(5)
97 CASE(6)
98 CASE(7)
99 CASE(8)
100 CASE(9)
101 CASE(10)
102 CASE(11)
103 CASE(12)
104 CASE(13)
105 CASE(14)
106 CASE(15)
107 CASE(16)
108 #undef CASE
109 default:
110 memcpy(dest, src, size);
111 return;
112 }
113 }
MemMove(void * dest,const void * src,size_t size)114 V8_EXPORT_PRIVATE inline void MemMove(void* dest, const void* src,
115 size_t size) {
116 // Fast path for small sizes. The compiler will expand the {memmove} for small
117 // fixed sizes to a sequence of move instructions. This avoids the overhead of
118 // the general {memmove} function.
119 switch (size) {
120 #define CASE(N) \
121 case N: \
122 memmove(dest, src, N); \
123 return;
124 CASE(1)
125 CASE(2)
126 CASE(3)
127 CASE(4)
128 CASE(5)
129 CASE(6)
130 CASE(7)
131 CASE(8)
132 CASE(9)
133 CASE(10)
134 CASE(11)
135 CASE(12)
136 CASE(13)
137 CASE(14)
138 CASE(15)
139 CASE(16)
140 #undef CASE
141 default:
142 memmove(dest, src, size);
143 return;
144 }
145 }
146 const size_t kMinComplexMemCopy = 8;
147 #endif // V8_TARGET_ARCH_IA32
148
149 // Copies words from |src| to |dst|. The data spans must not overlap.
150 // |src| and |dst| must be TWord-size aligned.
151 template <size_t kBlockCopyLimit, typename T>
CopyImpl(T * dst_ptr,const T * src_ptr,size_t count)152 inline void CopyImpl(T* dst_ptr, const T* src_ptr, size_t count) {
153 constexpr int kTWordSize = sizeof(T);
154 #ifdef DEBUG
155 Address dst = reinterpret_cast<Address>(dst_ptr);
156 Address src = reinterpret_cast<Address>(src_ptr);
157 DCHECK(IsAligned(dst, kTWordSize));
158 DCHECK(IsAligned(src, kTWordSize));
159 DCHECK(((src <= dst) && ((src + count * kTWordSize) <= dst)) ||
160 ((dst <= src) && ((dst + count * kTWordSize) <= src)));
161 #endif
162 if (count == 0) return;
163
164 // Use block copying MemCopy if the segment we're copying is
165 // enough to justify the extra call/setup overhead.
166 if (count < kBlockCopyLimit) {
167 do {
168 count--;
169 *dst_ptr++ = *src_ptr++;
170 } while (count > 0);
171 } else {
172 MemCopy(dst_ptr, src_ptr, count * kTWordSize);
173 }
174 }
175
176 // Copies kSystemPointerSize-sized words from |src| to |dst|. The data spans
177 // must not overlap. |src| and |dst| must be kSystemPointerSize-aligned.
CopyWords(Address dst,const Address src,size_t num_words)178 inline void CopyWords(Address dst, const Address src, size_t num_words) {
179 static const size_t kBlockCopyLimit = 16;
180 CopyImpl<kBlockCopyLimit>(reinterpret_cast<Address*>(dst),
181 reinterpret_cast<const Address*>(src), num_words);
182 }
183
184 // Copies data from |src| to |dst|. The data spans must not overlap.
185 template <typename T>
CopyBytes(T * dst,const T * src,size_t num_bytes)186 inline void CopyBytes(T* dst, const T* src, size_t num_bytes) {
187 STATIC_ASSERT(sizeof(T) == 1);
188 if (num_bytes == 0) return;
189 CopyImpl<kMinComplexMemCopy>(dst, src, num_bytes);
190 }
191
MemsetUint32(uint32_t * dest,uint32_t value,size_t counter)192 inline void MemsetUint32(uint32_t* dest, uint32_t value, size_t counter) {
193 #if V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
194 #define STOS "stosl"
195 #endif
196
197 #if defined(MEMORY_SANITIZER)
198 // MemorySanitizer does not understand inline assembly.
199 #undef STOS
200 #endif
201
202 #if defined(__GNUC__) && defined(STOS)
203 asm volatile(
204 "cld;"
205 "rep ; " STOS
206 : "+&c"(counter), "+&D"(dest)
207 : "a"(value)
208 : "memory", "cc");
209 #else
210 for (size_t i = 0; i < counter; i++) {
211 dest[i] = value;
212 }
213 #endif
214
215 #undef STOS
216 }
217
MemsetPointer(Address * dest,Address value,size_t counter)218 inline void MemsetPointer(Address* dest, Address value, size_t counter) {
219 #if V8_HOST_ARCH_IA32
220 #define STOS "stosl"
221 #elif V8_HOST_ARCH_X64
222 #define STOS "stosq"
223 #endif
224
225 #if defined(MEMORY_SANITIZER)
226 // MemorySanitizer does not understand inline assembly.
227 #undef STOS
228 #endif
229
230 #if defined(__GNUC__) && defined(STOS)
231 asm volatile(
232 "cld;"
233 "rep ; " STOS
234 : "+&c"(counter), "+&D"(dest)
235 : "a"(value)
236 : "memory", "cc");
237 #else
238 for (size_t i = 0; i < counter; i++) {
239 dest[i] = value;
240 }
241 #endif
242
243 #undef STOS
244 }
245
246 template <typename T, typename U>
MemsetPointer(T ** dest,U * value,size_t counter)247 inline void MemsetPointer(T** dest, U* value, size_t counter) {
248 #ifdef DEBUG
249 T* a = nullptr;
250 U* b = nullptr;
251 a = b; // Fake assignment to check assignability.
252 USE(a);
253 #endif // DEBUG
254 MemsetPointer(reinterpret_cast<Address*>(dest),
255 reinterpret_cast<Address>(value), counter);
256 }
257
258 template <typename T>
MemsetPointer(T ** dest,std::nullptr_t,size_t counter)259 inline void MemsetPointer(T** dest, std::nullptr_t, size_t counter) {
260 MemsetPointer(reinterpret_cast<Address*>(dest), Address{0}, counter);
261 }
262
263 // Copy from 8bit/16bit chars to 8bit/16bit chars. Values are zero-extended if
264 // needed. Ranges are not allowed to overlap.
265 // The separate declaration is needed for the V8_NONNULL, which is not allowed
266 // on a definition.
267 template <typename SrcType, typename DstType>
268 void CopyChars(DstType* dst, const SrcType* src, size_t count) V8_NONNULL(1, 2);
269
270 template <typename SrcType, typename DstType>
CopyChars(DstType * dst,const SrcType * src,size_t count)271 void CopyChars(DstType* dst, const SrcType* src, size_t count) {
272 STATIC_ASSERT(std::is_integral<SrcType>::value);
273 STATIC_ASSERT(std::is_integral<DstType>::value);
274 using SrcTypeUnsigned = typename std::make_unsigned<SrcType>::type;
275 using DstTypeUnsigned = typename std::make_unsigned<DstType>::type;
276
277 #ifdef DEBUG
278 // Check for no overlap, otherwise {std::copy_n} cannot be used.
279 Address src_start = reinterpret_cast<Address>(src);
280 Address src_end = src_start + count * sizeof(SrcType);
281 Address dst_start = reinterpret_cast<Address>(dst);
282 Address dst_end = dst_start + count * sizeof(DstType);
283 DCHECK(src_end <= dst_start || dst_end <= src_start);
284 #endif
285
286 auto* dst_u = reinterpret_cast<DstTypeUnsigned*>(dst);
287 auto* src_u = reinterpret_cast<const SrcTypeUnsigned*>(src);
288
289 // Especially Atom CPUs profit from this explicit instantiation for small
290 // counts. This gives up to 20 percent improvement for microbenchmarks such as
291 // joining an array of small integers (2019-10-16).
292 switch (count) {
293 #define CASE(N) \
294 case N: \
295 std::copy_n(src_u, N, dst_u); \
296 return;
297 CASE(1)
298 CASE(2)
299 CASE(3)
300 CASE(4)
301 CASE(5)
302 CASE(6)
303 CASE(7)
304 CASE(8)
305 CASE(9)
306 CASE(10)
307 CASE(11)
308 CASE(12)
309 CASE(13)
310 CASE(14)
311 CASE(15)
312 CASE(16)
313 #undef CASE
314 default:
315 std::copy_n(src_u, count, dst_u);
316 return;
317 }
318 }
319
320 } // namespace internal
321 } // namespace v8
322
323 #endif // V8_UTILS_MEMCOPY_H_
324