1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2020 Google LLC
4 */
5 #include <errno.h>
6 #include <fcntl.h>
7 #include <getopt.h>
8 #include <lz4.h>
9 #include <stdbool.h>
10 #include <stdint.h>
11 #include <stdio.h>
12 #include <stdlib.h>
13 #include <string.h>
14 #include <sys/mount.h>
15 #include <sys/stat.h>
16 #include <time.h>
17 #include <ctype.h>
18 #include <unistd.h>
19
20 #include "utils.h"
21
22 #define err_msg(...) \
23 do { \
24 fprintf(stderr, "%s: (%d) ", TAG, __LINE__); \
25 fprintf(stderr, __VA_ARGS__); \
26 fprintf(stderr, " (%s)\n", strerror(errno)); \
27 } while (false)
28
29 #define TAG "incfs_perf"
30
31 struct options {
32 int blocks; /* -b number of diff block sizes */
33 bool no_cleanup; /* -c don't clean up after */
34 const char *test_dir; /* -d working directory */
35 const char *file_types; /* -f sScCvV */
36 bool no_native; /* -n don't test native files */
37 bool no_random; /* -r don't do random reads*/
38 bool no_linear; /* -R random reads only */
39 size_t size; /* -s file size as power of 2 */
40 int tries; /* -t times to run test*/
41 };
42
43 enum flags {
44 SHUFFLE = 1,
45 COMPRESS = 2,
46 VERIFY = 4,
47 LAST_FLAG = 8,
48 };
49
print_help(void)50 void print_help(void)
51 {
52 puts(
53 "incfs_perf. Performance test tool for incfs\n"
54 "\tTests read performance of incfs by creating files of various types\n"
55 "\tflushing caches and then reading them back.\n"
56 "\tEach file is read with different block sizes and average\n"
57 "\tthroughput in megabytes/second and memory usage are reported for\n"
58 "\teach block size\n"
59 "\tNative files are tested for comparison\n"
60 "\tNative files are created in native folder, incfs files are created\n"
61 "\tin src folder which is mounted on dst folder\n"
62 "\n"
63 "\t-bn (default 8) number of different block sizes, starting at 4096\n"
64 "\t and doubling\n"
65 "\t-c don't Clean up - leave files and mount point\n"
66 "\t-d dir create directories in dir\n"
67 "\t-fs|Sc|Cv|V restrict which files are created.\n"
68 "\t s blocks not shuffled, S blocks shuffled\n"
69 "\t c blocks not compress, C blocks compressed\n"
70 "\t v files not verified, V files verified\n"
71 "\t If a letter is omitted, both options are tested\n"
72 "\t If no letter are given, incfs is not tested\n"
73 "\t-n Don't test native files\n"
74 "\t-r No random reads (sequential only)\n"
75 "\t-R Random reads only (no sequential)\n"
76 "\t-sn (default 30)File size as power of 2\n"
77 "\t-tn (default 5) Number of tries per file. Results are averaged\n"
78 );
79 }
80
parse_options(int argc,char * const * argv,struct options * options)81 int parse_options(int argc, char *const *argv, struct options *options)
82 {
83 signed char c;
84
85 /* Set defaults here */
86 *options = (struct options){
87 .blocks = 8,
88 .test_dir = ".",
89 .tries = 5,
90 .size = 30,
91 };
92
93 /* Load options from command line here */
94 while ((c = getopt(argc, argv, "b:cd:f::hnrRs:t:")) != -1) {
95 switch (c) {
96 case 'b':
97 options->blocks = strtol(optarg, NULL, 10);
98 break;
99
100 case 'c':
101 options->no_cleanup = true;
102 break;
103
104 case 'd':
105 options->test_dir = optarg;
106 break;
107
108 case 'f':
109 if (optarg)
110 options->file_types = optarg;
111 else
112 options->file_types = "sS";
113 break;
114
115 case 'h':
116 print_help();
117 exit(0);
118
119 case 'n':
120 options->no_native = true;
121 break;
122
123 case 'r':
124 options->no_random = true;
125 break;
126
127 case 'R':
128 options->no_linear = true;
129 break;
130
131 case 's':
132 options->size = strtol(optarg, NULL, 10);
133 break;
134
135 case 't':
136 options->tries = strtol(optarg, NULL, 10);
137 break;
138
139 default:
140 print_help();
141 return -EINVAL;
142 }
143 }
144
145 options->size = 1L << options->size;
146
147 return 0;
148 }
149
shuffle(size_t * buffer,size_t size)150 void shuffle(size_t *buffer, size_t size)
151 {
152 size_t i;
153
154 for (i = 0; i < size; ++i) {
155 size_t j = random() * (size - i - 1) / RAND_MAX;
156 size_t temp = buffer[i];
157
158 buffer[i] = buffer[j];
159 buffer[j] = temp;
160 }
161 }
162
get_free_memory(void)163 int get_free_memory(void)
164 {
165 FILE *meminfo = fopen("/proc/meminfo", "re");
166 char field[256];
167 char value[256] = {};
168
169 if (!meminfo)
170 return -ENOENT;
171
172 while (fscanf(meminfo, "%[^:]: %s kB\n", field, value) == 2) {
173 if (!strcmp(field, "MemFree"))
174 break;
175 *value = 0;
176 }
177
178 fclose(meminfo);
179
180 if (!*value)
181 return -ENOENT;
182
183 return strtol(value, NULL, 10);
184 }
185
write_data(int cmd_fd,int dir_fd,const char * name,size_t size,int flags)186 int write_data(int cmd_fd, int dir_fd, const char *name, size_t size, int flags)
187 {
188 int fd = openat(dir_fd, name, O_RDWR | O_CLOEXEC);
189 struct incfs_permit_fill permit_fill = {
190 .file_descriptor = fd,
191 };
192 int block_count = 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
193 size_t *blocks = malloc(sizeof(size_t) * block_count);
194 int error = 0;
195 size_t i;
196 uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
197 uint8_t compressed_data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
198 struct incfs_fill_block fill_block = {
199 .compression = COMPRESSION_NONE,
200 .data_len = sizeof(data),
201 .data = ptr_to_u64(data),
202 };
203
204 if (!blocks) {
205 err_msg("Out of memory");
206 error = -errno;
207 goto out;
208 }
209
210 if (fd == -1) {
211 err_msg("Could not open file for writing %s", name);
212 error = -errno;
213 goto out;
214 }
215
216 if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
217 err_msg("Failed to call PERMIT_FILL");
218 error = -errno;
219 goto out;
220 }
221
222 for (i = 0; i < block_count; ++i)
223 blocks[i] = i;
224
225 if (flags & SHUFFLE)
226 shuffle(blocks, block_count);
227
228 if (flags & COMPRESS) {
229 size_t comp_size = LZ4_compress_default(
230 (char *)data, (char *)compressed_data, sizeof(data),
231 ARRAY_SIZE(compressed_data));
232
233 if (comp_size <= 0) {
234 error = -EBADMSG;
235 goto out;
236 }
237 fill_block.compression = COMPRESSION_LZ4;
238 fill_block.data = ptr_to_u64(compressed_data);
239 fill_block.data_len = comp_size;
240 }
241
242 for (i = 0; i < block_count; ++i) {
243 struct incfs_fill_blocks fill_blocks = {
244 .count = 1,
245 .fill_blocks = ptr_to_u64(&fill_block),
246 };
247
248 fill_block.block_index = blocks[i];
249 int written = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
250
251 if (written != 1) {
252 error = -errno;
253 err_msg("Failed to write block %lu in file %s", i,
254 name);
255 break;
256 }
257 }
258
259 out:
260 free(blocks);
261 close(fd);
262 sync();
263 return error;
264 }
265
measure_read_throughput_internal(const char * tag,int dir,const char * name,const struct options * options,bool random)266 int measure_read_throughput_internal(const char *tag, int dir, const char *name,
267 const struct options *options, bool random)
268 {
269 int block;
270
271 if (random)
272 printf("%32s(random)", tag);
273 else
274 printf("%40s", tag);
275
276 for (block = 0; block < options->blocks; ++block) {
277 size_t buffer_size;
278 char *buffer;
279 int try;
280 double time = 0;
281 double throughput;
282 int memory = 0;
283
284 buffer_size = 1 << (block + 12);
285 buffer = malloc(buffer_size);
286
287 for (try = 0; try < options->tries; ++try) {
288 int err;
289 struct timespec start_time, end_time;
290 off_t i;
291 int fd;
292 size_t offsets_size = options->size / buffer_size;
293 size_t *offsets =
294 malloc(offsets_size * sizeof(*offsets));
295 int start_memory, end_memory;
296
297 if (!offsets) {
298 err_msg("Not enough memory");
299 return -ENOMEM;
300 }
301
302 for (i = 0; i < offsets_size; ++i)
303 offsets[i] = i * buffer_size;
304
305 if (random)
306 shuffle(offsets, offsets_size);
307
308 err = drop_caches();
309 if (err) {
310 err_msg("Failed to drop caches");
311 return err;
312 }
313
314 start_memory = get_free_memory();
315 if (start_memory < 0) {
316 err_msg("Failed to get start memory");
317 return start_memory;
318 }
319
320 fd = openat(dir, name, O_RDONLY | O_CLOEXEC);
321 if (fd == -1) {
322 err_msg("Failed to open file");
323 return err;
324 }
325
326 err = clock_gettime(CLOCK_MONOTONIC, &start_time);
327 if (err) {
328 err_msg("Failed to get start time");
329 return err;
330 }
331
332 for (i = 0; i < offsets_size; ++i)
333 if (pread(fd, buffer, buffer_size,
334 offsets[i]) != buffer_size) {
335 err_msg("Failed to read file");
336 err = -errno;
337 goto fail;
338 }
339
340 err = clock_gettime(CLOCK_MONOTONIC, &end_time);
341 if (err) {
342 err_msg("Failed to get start time");
343 goto fail;
344 }
345
346 end_memory = get_free_memory();
347 if (end_memory < 0) {
348 err_msg("Failed to get end memory");
349 return end_memory;
350 }
351
352 time += end_time.tv_sec - start_time.tv_sec;
353 time += (end_time.tv_nsec - start_time.tv_nsec) / 1e9;
354
355 close(fd);
356 fd = -1;
357 memory += start_memory - end_memory;
358
359 fail:
360 free(offsets);
361 close(fd);
362 if (err)
363 return err;
364 }
365
366 throughput = options->size * options->tries / time;
367 printf("%10.3e %10d", throughput, memory / options->tries);
368 free(buffer);
369 }
370
371 printf("\n");
372 return 0;
373 }
374
measure_read_throughput(const char * tag,int dir,const char * name,const struct options * options)375 int measure_read_throughput(const char *tag, int dir, const char *name,
376 const struct options *options)
377 {
378 int err = 0;
379
380 if (!options->no_linear)
381 err = measure_read_throughput_internal(tag, dir, name, options,
382 false);
383
384 if (!err && !options->no_random)
385 err = measure_read_throughput_internal(tag, dir, name, options,
386 true);
387 return err;
388 }
389
test_native_file(int dir,const struct options * options)390 int test_native_file(int dir, const struct options *options)
391 {
392 const char *name = "file";
393 int fd;
394 char buffer[4096] = {};
395 off_t i;
396 int err;
397
398 fd = openat(dir, name, O_CREAT | O_WRONLY | O_CLOEXEC, 0600);
399 if (fd == -1) {
400 err_msg("Could not open native file");
401 return -errno;
402 }
403
404 for (i = 0; i < options->size; i += sizeof(buffer))
405 if (pwrite(fd, buffer, sizeof(buffer), i) != sizeof(buffer)) {
406 err_msg("Failed to write file");
407 err = -errno;
408 goto fail;
409 }
410
411 close(fd);
412 sync();
413 fd = -1;
414
415 err = measure_read_throughput("native", dir, name, options);
416
417 fail:
418 close(fd);
419 return err;
420 }
421
422 struct hash_block {
423 char data[INCFS_DATA_FILE_BLOCK_SIZE];
424 };
425
build_mtree(size_t size,char * root_hash,int * mtree_block_count)426 static struct hash_block *build_mtree(size_t size, char *root_hash,
427 int *mtree_block_count)
428 {
429 char data[INCFS_DATA_FILE_BLOCK_SIZE] = {};
430 const int digest_size = SHA256_DIGEST_SIZE;
431 const int hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
432 int block_count = 0;
433 int hash_block_count = 0;
434 int total_tree_block_count = 0;
435 int tree_lvl_index[INCFS_MAX_MTREE_LEVELS] = {};
436 int tree_lvl_count[INCFS_MAX_MTREE_LEVELS] = {};
437 int levels_count = 0;
438 int i, level;
439 struct hash_block *mtree;
440
441 if (size == 0)
442 return 0;
443
444 block_count = 1 + (size - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
445 hash_block_count = block_count;
446 for (i = 0; hash_block_count > 1; i++) {
447 hash_block_count = (hash_block_count + hash_per_block - 1) /
448 hash_per_block;
449 tree_lvl_count[i] = hash_block_count;
450 total_tree_block_count += hash_block_count;
451 }
452 levels_count = i;
453
454 for (i = 0; i < levels_count; i++) {
455 int prev_lvl_base = (i == 0) ? total_tree_block_count :
456 tree_lvl_index[i - 1];
457
458 tree_lvl_index[i] = prev_lvl_base - tree_lvl_count[i];
459 }
460
461 *mtree_block_count = total_tree_block_count;
462 mtree = calloc(total_tree_block_count, sizeof(*mtree));
463 /* Build level 0 hashes. */
464 for (i = 0; i < block_count; i++) {
465 int block_index = tree_lvl_index[0] + i / hash_per_block;
466 int block_off = (i % hash_per_block) * digest_size;
467 char *hash_ptr = mtree[block_index].data + block_off;
468
469 sha256(data, INCFS_DATA_FILE_BLOCK_SIZE, hash_ptr);
470 }
471
472 /* Build higher levels of hash tree. */
473 for (level = 1; level < levels_count; level++) {
474 int prev_lvl_base = tree_lvl_index[level - 1];
475 int prev_lvl_count = tree_lvl_count[level - 1];
476
477 for (i = 0; i < prev_lvl_count; i++) {
478 int block_index =
479 i / hash_per_block + tree_lvl_index[level];
480 int block_off = (i % hash_per_block) * digest_size;
481 char *hash_ptr = mtree[block_index].data + block_off;
482
483 sha256(mtree[i + prev_lvl_base].data,
484 INCFS_DATA_FILE_BLOCK_SIZE, hash_ptr);
485 }
486 }
487
488 /* Calculate root hash from the top block */
489 sha256(mtree[0].data, INCFS_DATA_FILE_BLOCK_SIZE, root_hash);
490
491 return mtree;
492 }
493
load_hash_tree(int cmd_fd,int dir,const char * name,struct hash_block * mtree,int mtree_block_count)494 static int load_hash_tree(int cmd_fd, int dir, const char *name,
495 struct hash_block *mtree, int mtree_block_count)
496 {
497 int err;
498 int i;
499 int fd;
500 struct incfs_fill_block *fill_block_array =
501 calloc(mtree_block_count, sizeof(struct incfs_fill_block));
502 struct incfs_fill_blocks fill_blocks = {
503 .count = mtree_block_count,
504 .fill_blocks = ptr_to_u64(fill_block_array),
505 };
506 struct incfs_permit_fill permit_fill;
507
508 if (!fill_block_array)
509 return -ENOMEM;
510
511 for (i = 0; i < fill_blocks.count; i++) {
512 fill_block_array[i] = (struct incfs_fill_block){
513 .block_index = i,
514 .data_len = INCFS_DATA_FILE_BLOCK_SIZE,
515 .data = ptr_to_u64(mtree[i].data),
516 .flags = INCFS_BLOCK_FLAGS_HASH
517 };
518 }
519
520 fd = openat(dir, name, O_RDONLY | O_CLOEXEC);
521 if (fd < 0) {
522 err = errno;
523 goto failure;
524 }
525
526 permit_fill.file_descriptor = fd;
527 if (ioctl(cmd_fd, INCFS_IOC_PERMIT_FILL, &permit_fill)) {
528 err_msg("Failed to call PERMIT_FILL");
529 err = -errno;
530 goto failure;
531 }
532
533 err = ioctl(fd, INCFS_IOC_FILL_BLOCKS, &fill_blocks);
534 close(fd);
535 if (err < fill_blocks.count)
536 err = errno;
537 else
538 err = 0;
539
540 failure:
541 free(fill_block_array);
542 return err;
543 }
544
test_incfs_file(int dst_dir,const struct options * options,int flags)545 int test_incfs_file(int dst_dir, const struct options *options, int flags)
546 {
547 int cmd_file = openat(dst_dir, INCFS_PENDING_READS_FILENAME,
548 O_RDONLY | O_CLOEXEC);
549 int err;
550 char name[4];
551 incfs_uuid_t id;
552 char tag[256];
553
554 snprintf(name, sizeof(name), "%c%c%c",
555 flags & SHUFFLE ? 'S' : 's',
556 flags & COMPRESS ? 'C' : 'c',
557 flags & VERIFY ? 'V' : 'v');
558
559 if (cmd_file == -1) {
560 err_msg("Could not open command file");
561 return -errno;
562 }
563
564 if (flags & VERIFY) {
565 char root_hash[INCFS_MAX_HASH_SIZE];
566 int mtree_block_count;
567 struct hash_block *mtree = build_mtree(options->size, root_hash,
568 &mtree_block_count);
569
570 if (!mtree) {
571 err_msg("Failed to build hash tree");
572 err = -ENOMEM;
573 goto fail;
574 }
575
576 err = crypto_emit_file(cmd_file, NULL, name, &id, options->size,
577 root_hash, "add_data");
578
579 if (!err)
580 err = load_hash_tree(cmd_file, dst_dir, name, mtree,
581 mtree_block_count);
582
583 free(mtree);
584 } else
585 err = emit_file(cmd_file, NULL, name, &id, options->size, NULL);
586
587 if (err) {
588 err_msg("Failed to create file %s", name);
589 goto fail;
590 }
591
592 if (write_data(cmd_file, dst_dir, name, options->size, flags))
593 goto fail;
594
595 snprintf(tag, sizeof(tag), "incfs%s%s%s",
596 flags & SHUFFLE ? "(shuffle)" : "",
597 flags & COMPRESS ? "(compress)" : "",
598 flags & VERIFY ? "(verify)" : "");
599
600 err = measure_read_throughput(tag, dst_dir, name, options);
601
602 fail:
603 close(cmd_file);
604 return err;
605 }
606
skip(struct options const * options,int flag,char c)607 bool skip(struct options const *options, int flag, char c)
608 {
609 if (!options->file_types)
610 return false;
611
612 if (flag && strchr(options->file_types, tolower(c)))
613 return true;
614
615 if (!flag && strchr(options->file_types, toupper(c)))
616 return true;
617
618 return false;
619 }
620
main(int argc,char * const * argv)621 int main(int argc, char *const *argv)
622 {
623 struct options options;
624 int err;
625 const char *native_dir = "native";
626 const char *src_dir = "src";
627 const char *dst_dir = "dst";
628 int native_dir_fd = -1;
629 int src_dir_fd = -1;
630 int dst_dir_fd = -1;
631 int block;
632 int flags;
633
634 err = parse_options(argc, argv, &options);
635 if (err)
636 return err;
637
638 err = chdir(options.test_dir);
639 if (err) {
640 err_msg("Failed to change to %s", options.test_dir);
641 return -errno;
642 }
643
644 /* Clean up any interrupted previous runs */
645 while (!umount(dst_dir))
646 ;
647
648 err = remove_dir(native_dir) || remove_dir(src_dir) ||
649 remove_dir(dst_dir);
650 if (err)
651 return err;
652
653 err = mkdir(native_dir, 0700);
654 if (err) {
655 err_msg("Failed to make directory %s", src_dir);
656 err = -errno;
657 goto cleanup;
658 }
659
660 err = mkdir(src_dir, 0700);
661 if (err) {
662 err_msg("Failed to make directory %s", src_dir);
663 err = -errno;
664 goto cleanup;
665 }
666
667 err = mkdir(dst_dir, 0700);
668 if (err) {
669 err_msg("Failed to make directory %s", src_dir);
670 err = -errno;
671 goto cleanup;
672 }
673
674 err = mount_fs_opt(dst_dir, src_dir, "readahead=0,rlog_pages=0", 0);
675 if (err) {
676 err_msg("Failed to mount incfs");
677 goto cleanup;
678 }
679
680 native_dir_fd = open(native_dir, O_RDONLY | O_CLOEXEC);
681 src_dir_fd = open(src_dir, O_RDONLY | O_CLOEXEC);
682 dst_dir_fd = open(dst_dir, O_RDONLY | O_CLOEXEC);
683 if (native_dir_fd == -1 || src_dir_fd == -1 || dst_dir_fd == -1) {
684 err_msg("Failed to open native, src or dst dir");
685 err = -errno;
686 goto cleanup;
687 }
688
689 printf("%40s", "");
690 for (block = 0; block < options.blocks; ++block)
691 printf("%21d", 1 << (block + 12));
692 printf("\n");
693
694 if (!err && !options.no_native)
695 err = test_native_file(native_dir_fd, &options);
696
697 for (flags = 0; flags < LAST_FLAG && !err; ++flags) {
698 if (skip(&options, flags & SHUFFLE, 's') ||
699 skip(&options, flags & COMPRESS, 'c') ||
700 skip(&options, flags & VERIFY, 'v'))
701 continue;
702 err = test_incfs_file(dst_dir_fd, &options, flags);
703 }
704
705 cleanup:
706 close(native_dir_fd);
707 close(src_dir_fd);
708 close(dst_dir_fd);
709 if (!options.no_cleanup) {
710 umount(dst_dir);
711 remove_dir(native_dir);
712 remove_dir(dst_dir);
713 remove_dir(src_dir);
714 }
715
716 return err;
717 }
718