1 /*
2 * memcpy benchmark.
3 *
4 * Copyright (c) 2020-2022, Arm Limited.
5 * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
6 */
7
8 #define _GNU_SOURCE
9 #include <stdint.h>
10 #include <stdio.h>
11 #include <string.h>
12 #include <assert.h>
13 #include "stringlib.h"
14 #include "benchlib.h"
15
16 #define ITERS 5000
17 #define ITERS2 20000000
18 #define ITERS3 200000
19 #define NUM_TESTS 16384
20 #define MIN_SIZE 32768
21 #define MAX_SIZE (1024 * 1024)
22
23 static uint8_t a[MAX_SIZE + 4096 + 64] __attribute__((__aligned__(64)));
24 static uint8_t b[MAX_SIZE + 4096 + 64] __attribute__((__aligned__(64)));
25
26 #define F(x) {#x, x},
27
28 static const struct fun
29 {
30 const char *name;
31 void *(*fun)(void *, const void *, size_t);
32 } funtab[] =
33 {
34 #if __aarch64__
35 F(__memcpy_aarch64)
36 # if __ARM_NEON
37 F(__memcpy_aarch64_simd)
38 # endif
39 # if __ARM_FEATURE_SVE
40 F(__memcpy_aarch64_sve)
41 # endif
42 #elif __arm__
43 F(__memcpy_arm)
44 #endif
45 F(memcpy)
46 #undef F
47 {0, 0}
48 };
49
50 typedef struct { uint16_t size; uint16_t freq; } freq_data_t;
51 typedef struct { uint8_t align; uint16_t freq; } align_data_t;
52
53 #define SIZE_NUM 65536
54 #define SIZE_MASK (SIZE_NUM-1)
55 static uint8_t size_arr[SIZE_NUM];
56
57 /* Frequency data for memcpy of less than 4096 bytes based on SPEC2017. */
58 static freq_data_t size_freq[] =
59 {
60 {32,22320}, { 16,9554}, { 8,8915}, {152,5327}, { 4,2159}, {292,2035},
61 { 12,1608}, { 24,1343}, {1152,895}, {144, 813}, {884, 733}, {284, 721},
62 {120, 661}, { 2, 649}, {882, 550}, { 5, 475}, { 7, 461}, {108, 460},
63 { 10, 361}, { 9, 361}, { 6, 334}, { 3, 326}, {464, 308}, {2048,303},
64 { 1, 298}, { 64, 250}, { 11, 197}, {296, 194}, { 68, 187}, { 15, 185},
65 {192, 184}, {1764,183}, { 13, 173}, {560, 126}, {160, 115}, {288, 96},
66 {104, 96}, {1144, 83}, { 18, 80}, { 23, 78}, { 40, 77}, { 19, 68},
67 { 48, 63}, { 17, 57}, { 72, 54}, {1280, 51}, { 20, 49}, { 28, 47},
68 { 22, 46}, {640, 45}, { 25, 41}, { 14, 40}, { 56, 37}, { 27, 35},
69 { 35, 33}, {384, 33}, { 29, 32}, { 80, 30}, {4095, 22}, {232, 22},
70 { 36, 19}, {184, 17}, { 21, 17}, {256, 16}, { 44, 15}, { 26, 15},
71 { 31, 14}, { 88, 14}, {176, 13}, { 33, 12}, {1024, 12}, {208, 11},
72 { 62, 11}, {128, 10}, {704, 10}, {324, 10}, { 96, 10}, { 60, 9},
73 {136, 9}, {124, 9}, { 34, 8}, { 30, 8}, {480, 8}, {1344, 8},
74 {273, 7}, {520, 7}, {112, 6}, { 52, 6}, {344, 6}, {336, 6},
75 {504, 5}, {168, 5}, {424, 5}, { 0, 4}, { 76, 3}, {200, 3},
76 {512, 3}, {312, 3}, {240, 3}, {960, 3}, {264, 2}, {672, 2},
77 { 38, 2}, {328, 2}, { 84, 2}, { 39, 2}, {216, 2}, { 42, 2},
78 { 37, 2}, {1608, 2}, { 70, 2}, { 46, 2}, {536, 2}, {280, 1},
79 {248, 1}, { 47, 1}, {1088, 1}, {1288, 1}, {224, 1}, { 41, 1},
80 { 50, 1}, { 49, 1}, {808, 1}, {360, 1}, {440, 1}, { 43, 1},
81 { 45, 1}, { 78, 1}, {968, 1}, {392, 1}, { 54, 1}, { 53, 1},
82 { 59, 1}, {376, 1}, {664, 1}, { 58, 1}, {272, 1}, { 66, 1},
83 {2688, 1}, {472, 1}, {568, 1}, {720, 1}, { 51, 1}, { 63, 1},
84 { 86, 1}, {496, 1}, {776, 1}, { 57, 1}, {680, 1}, {792, 1},
85 {122, 1}, {760, 1}, {824, 1}, {552, 1}, { 67, 1}, {456, 1},
86 {984, 1}, { 74, 1}, {408, 1}, { 75, 1}, { 92, 1}, {576, 1},
87 {116, 1}, { 65, 1}, {117, 1}, { 82, 1}, {352, 1}, { 55, 1},
88 {100, 1}, { 90, 1}, {696, 1}, {111, 1}, {880, 1}, { 79, 1},
89 {488, 1}, { 61, 1}, {114, 1}, { 94, 1}, {1032, 1}, { 98, 1},
90 { 87, 1}, {584, 1}, { 85, 1}, {648, 1}, {0, 0}
91 };
92
93 #define ALIGN_NUM 1024
94 #define ALIGN_MASK (ALIGN_NUM-1)
95 static uint8_t src_align_arr[ALIGN_NUM];
96 static uint8_t dst_align_arr[ALIGN_NUM];
97
98 /* Source alignment frequency for memcpy based on SPEC2017. */
99 static align_data_t src_align_freq[] =
100 {
101 {8, 300}, {16, 292}, {32, 168}, {64, 153}, {4, 79}, {2, 14}, {1, 18}, {0, 0}
102 };
103
104 static align_data_t dst_align_freq[] =
105 {
106 {8, 265}, {16, 263}, {64, 209}, {32, 174}, {4, 90}, {2, 10}, {1, 13}, {0, 0}
107 };
108
109 typedef struct
110 {
111 uint64_t src : 24;
112 uint64_t dst : 24;
113 uint64_t len : 16;
114 } copy_t;
115
116 static copy_t test_arr[NUM_TESTS];
117
118 typedef char *(*proto_t) (char *, const char *, size_t);
119
120 static void
init_copy_distribution(void)121 init_copy_distribution (void)
122 {
123 int i, j, freq, size, n;
124
125 for (n = i = 0; (freq = size_freq[i].freq) != 0; i++)
126 for (j = 0, size = size_freq[i].size; j < freq; j++)
127 size_arr[n++] = size;
128 assert (n == SIZE_NUM);
129
130 for (n = i = 0; (freq = src_align_freq[i].freq) != 0; i++)
131 for (j = 0, size = src_align_freq[i].align; j < freq; j++)
132 src_align_arr[n++] = size - 1;
133 assert (n == ALIGN_NUM);
134
135 for (n = i = 0; (freq = dst_align_freq[i].freq) != 0; i++)
136 for (j = 0, size = dst_align_freq[i].align; j < freq; j++)
137 dst_align_arr[n++] = size - 1;
138 assert (n == ALIGN_NUM);
139 }
140
141 static size_t
init_copies(size_t max_size)142 init_copies (size_t max_size)
143 {
144 size_t total = 0;
145 /* Create a random set of copies with the given size and alignment
146 distributions. */
147 for (int i = 0; i < NUM_TESTS; i++)
148 {
149 test_arr[i].dst = (rand32 (0) & (max_size - 1));
150 test_arr[i].dst &= ~dst_align_arr[rand32 (0) & ALIGN_MASK];
151 test_arr[i].src = (rand32 (0) & (max_size - 1));
152 test_arr[i].src &= ~src_align_arr[rand32 (0) & ALIGN_MASK];
153 test_arr[i].len = size_arr[rand32 (0) & SIZE_MASK];
154 total += test_arr[i].len;
155 }
156
157 return total;
158 }
159
main(void)160 int main (void)
161 {
162 init_copy_distribution ();
163
164 memset (a, 1, sizeof (a));
165 memset (b, 2, sizeof (b));
166
167 printf("Random memcpy (bytes/ns):\n");
168 for (int f = 0; funtab[f].name != 0; f++)
169 {
170 size_t total = 0;
171 uint64_t tsum = 0;
172 printf ("%22s ", funtab[f].name);
173 rand32 (0x12345678);
174
175 for (int size = MIN_SIZE; size <= MAX_SIZE; size *= 2)
176 {
177 size_t copy_size = init_copies (size) * ITERS;
178
179 for (int c = 0; c < NUM_TESTS; c++)
180 funtab[f].fun (b + test_arr[c].dst, a + test_arr[c].src,
181 test_arr[c].len);
182
183 uint64_t t = clock_get_ns ();
184 for (int i = 0; i < ITERS; i++)
185 for (int c = 0; c < NUM_TESTS; c++)
186 funtab[f].fun (b + test_arr[c].dst, a + test_arr[c].src,
187 test_arr[c].len);
188 t = clock_get_ns () - t;
189 total += copy_size;
190 tsum += t;
191 printf ("%dK: %.2f ", size / 1024, (double)copy_size / t);
192 }
193 printf( "avg %.2f\n", (double)total / tsum);
194 }
195
196 size_t total = 0;
197 uint64_t tsum = 0;
198 printf ("%22s ", "memcpy_call");
199 rand32 (0x12345678);
200
201 for (int size = MIN_SIZE; size <= MAX_SIZE; size *= 2)
202 {
203 size_t copy_size = init_copies (size) * ITERS;
204
205 for (int c = 0; c < NUM_TESTS; c++)
206 memcpy (b + test_arr[c].dst, a + test_arr[c].src, test_arr[c].len);
207
208 uint64_t t = clock_get_ns ();
209 for (int i = 0; i < ITERS; i++)
210 for (int c = 0; c < NUM_TESTS; c++)
211 memcpy (b + test_arr[c].dst, a + test_arr[c].src, test_arr[c].len);
212 t = clock_get_ns () - t;
213 total += copy_size;
214 tsum += t;
215 printf ("%dK: %.2f ", size / 1024, (double)copy_size / t);
216 }
217 printf( "avg %.2f\n", (double)total / tsum);
218
219
220 printf ("\nAligned medium memcpy (bytes/ns):\n");
221 for (int f = 0; funtab[f].name != 0; f++)
222 {
223 printf ("%22s ", funtab[f].name);
224
225 for (int size = 8; size <= 512; size *= 2)
226 {
227 uint64_t t = clock_get_ns ();
228 for (int i = 0; i < ITERS2; i++)
229 funtab[f].fun (b, a, size);
230 t = clock_get_ns () - t;
231 printf ("%dB: %.2f ", size, (double)size * ITERS2 / t);
232 }
233 printf ("\n");
234 }
235
236 printf ("%22s ", "memcpy_call");
237 for (int size = 8; size <= 512; size *= 2)
238 {
239 uint64_t t = clock_get_ns ();
240 for (int i = 0; i < ITERS2; i++)
241 memcpy (b, a, size);
242 t = clock_get_ns () - t;
243 printf ("%dB: %.2f ", size, (double)size * ITERS2 / t);
244 }
245 printf ("\n");
246
247
248 printf ("\nUnaligned medium memcpy (bytes/ns):\n");
249 for (int f = 0; funtab[f].name != 0; f++)
250 {
251 printf ("%22s ", funtab[f].name);
252
253 for (int size = 8; size <= 512; size *= 2)
254 {
255 uint64_t t = clock_get_ns ();
256 for (int i = 0; i < ITERS2; i++)
257 funtab[f].fun (b + 3, a + 1, size);
258 t = clock_get_ns () - t;
259 printf ("%dB: %.2f ", size, (double)size * ITERS2 / t);
260 }
261 printf ("\n");
262 }
263
264 printf ("%22s ", "memcpy_call");
265 for (int size = 8; size <= 512; size *= 2)
266 {
267 uint64_t t = clock_get_ns ();
268 for (int i = 0; i < ITERS2; i++)
269 memcpy (b + 3, a + 1, size);
270 t = clock_get_ns () - t;
271 printf ("%dB: %.2f ", size, (double)size * ITERS2 / t);
272 }
273 printf ("\n");
274
275
276 printf ("\nLarge memcpy (bytes/ns):\n");
277 for (int f = 0; funtab[f].name != 0; f++)
278 {
279 printf ("%22s ", funtab[f].name);
280
281 for (int size = 1024; size <= 65536; size *= 2)
282 {
283 uint64_t t = clock_get_ns ();
284 for (int i = 0; i < ITERS3; i++)
285 funtab[f].fun (b, a, size);
286 t = clock_get_ns () - t;
287 printf ("%dK: %.2f ", size / 1024, (double)size * ITERS3 / t);
288 }
289 printf ("\n");
290 }
291
292 printf ("%22s ", "memcpy_call");
293 for (int size = 1024; size <= 65536; size *= 2)
294 {
295 uint64_t t = clock_get_ns ();
296 for (int i = 0; i < ITERS3; i++)
297 memcpy (b, a, size);
298 t = clock_get_ns () - t;
299 printf ("%dK: %.2f ", size / 1024, (double)size * ITERS3 / t);
300 }
301 printf ("\n");
302
303
304 printf ("\nUnaligned forwards memmove (bytes/ns):\n");
305 for (int f = 0; funtab[f].name != 0; f++)
306 {
307 printf ("%22s ", funtab[f].name);
308
309 for (int size = 1024; size <= 65536; size *= 2)
310 {
311 uint64_t t = clock_get_ns ();
312 for (int i = 0; i < ITERS3; i++)
313 funtab[f].fun (a, a + 256 + (i & 31), size);
314 t = clock_get_ns () - t;
315 printf ("%dK: %.2f ", size / 1024, (double)size * ITERS3 / t);
316 }
317 printf ("\n");
318 }
319
320
321 printf ("\nUnaligned backwards memmove (bytes/ns):\n");
322 for (int f = 0; funtab[f].name != 0; f++)
323 {
324 printf ("%22s ", funtab[f].name);
325
326 for (int size = 1024; size <= 65536; size *= 2)
327 {
328 uint64_t t = clock_get_ns ();
329 for (int i = 0; i < ITERS3; i++)
330 funtab[f].fun (a + 256 + (i & 31), a, size);
331 t = clock_get_ns () - t;
332 printf ("%dK: %.2f ", size / 1024, (double)size * ITERS3 / t);
333 }
334 printf ("\n");
335 }
336 printf ("\n");
337
338 return 0;
339 }
340