1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Copyright (C) 2018-2019 HUAWEI, Inc.
4 * http://www.huawei.com/
5 * Created by Li Guifu <bluce.liguifu@huawei.com>
6 */
7 #define _GNU_SOURCE
8 #include <time.h>
9 #include <sys/time.h>
10 #include <stdlib.h>
11 #include <limits.h>
12 #include <libgen.h>
13 #include <sys/stat.h>
14 #include <getopt.h>
15 #include "erofs/config.h"
16 #include "erofs/print.h"
17 #include "erofs/cache.h"
18 #include "erofs/diskbuf.h"
19 #include "erofs/inode.h"
20 #include "erofs/tar.h"
21 #include "erofs/io.h"
22 #include "erofs/compress.h"
23 #include "erofs/dedupe.h"
24 #include "erofs/xattr.h"
25 #include "erofs/exclude.h"
26 #include "erofs/block_list.h"
27 #include "erofs/compress_hints.h"
28 #include "erofs/blobchunk.h"
29 #include "erofs/fragments.h"
30 #include "erofs/rebuild.h"
31 #include "../lib/liberofs_private.h"
32 #include "../lib/liberofs_uuid.h"
33
34 #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
35
36 static struct option long_options[] = {
37 {"help", no_argument, 0, 1},
38 {"exclude-path", required_argument, NULL, 2},
39 {"exclude-regex", required_argument, NULL, 3},
40 #ifdef HAVE_LIBSELINUX
41 {"file-contexts", required_argument, NULL, 4},
42 #endif
43 {"force-uid", required_argument, NULL, 5},
44 {"force-gid", required_argument, NULL, 6},
45 {"all-root", no_argument, NULL, 7},
46 #ifndef NDEBUG
47 {"random-pclusterblks", no_argument, NULL, 8},
48 {"random-algorithms", no_argument, NULL, 18},
49 #endif
50 {"max-extent-bytes", required_argument, NULL, 9},
51 {"compress-hints", required_argument, NULL, 10},
52 {"chunksize", required_argument, NULL, 11},
53 {"quiet", no_argument, 0, 12},
54 {"blobdev", required_argument, NULL, 13},
55 {"ignore-mtime", no_argument, NULL, 14},
56 {"preserve-mtime", no_argument, NULL, 15},
57 {"uid-offset", required_argument, NULL, 16},
58 {"gid-offset", required_argument, NULL, 17},
59 {"tar", optional_argument, NULL, 20},
60 {"aufs", no_argument, NULL, 21},
61 {"mount-point", required_argument, NULL, 512},
62 {"xattr-prefix", required_argument, NULL, 19},
63 #ifdef WITH_ANDROID
64 {"product-out", required_argument, NULL, 513},
65 {"fs-config-file", required_argument, NULL, 514},
66 {"block-list-file", required_argument, NULL, 515},
67 #endif
68 {"ovlfs-strip", optional_argument, NULL, 516},
69 #ifdef HAVE_ZLIB
70 {"gzip", no_argument, NULL, 517},
71 #endif
72 {0, 0, 0, 0},
73 };
74
print_available_compressors(FILE * f,const char * delim)75 static void print_available_compressors(FILE *f, const char *delim)
76 {
77 int i = 0;
78 bool comma = false;
79 const char *s;
80
81 while ((s = z_erofs_list_available_compressors(&i)) != NULL) {
82 if (comma)
83 fputs(delim, f);
84 fputs(s, f);
85 comma = true;
86 }
87 fputc('\n', f);
88 }
89
usage(void)90 static void usage(void)
91 {
92 fputs("usage: [options] FILE SOURCE(s)\n"
93 "Generate EROFS image (FILE) from DIRECTORY, TARBALL and/or EROFS images. And [options] are:\n"
94 " -b# set block size to # (# = page size by default)\n"
95 " -d# set output message level to # (maximum 9)\n"
96 " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
97 " -zX[,Y][:..] X=compressor (Y=compression level, optional)\n"
98 " alternative algorithms can be separated by colons(:)\n"
99 " -C# specify the size of compress physical cluster in bytes\n"
100 " -EX[,...] X=extended options\n"
101 " -L volume-label set the volume label (maximum 16)\n"
102 " -T# set a fixed UNIX timestamp # to all files\n"
103 " -UX use a given filesystem UUID\n"
104 " --all-root make all files owned by root\n"
105 " --blobdev=X specify an extra device X to store chunked data\n"
106 " --chunksize=# generate chunk-based files with #-byte chunks\n"
107 " --compress-hints=X specify a file to configure per-file compression strategy\n"
108 " --exclude-path=X avoid including file X (X = exact literal path)\n"
109 " --exclude-regex=X avoid including files that match X (X = regular expression)\n"
110 #ifdef HAVE_LIBSELINUX
111 " --file-contexts=X specify a file contexts file to setup selinux labels\n"
112 #endif
113 " --force-uid=# set all file uids to # (# = UID)\n"
114 " --force-gid=# set all file gids to # (# = GID)\n"
115 " --uid-offset=# add offset # to all file uids (# = id offset)\n"
116 " --gid-offset=# add offset # to all file gids (# = id offset)\n"
117 #ifdef HAVE_ZLIB
118 " --gzip try to filter the tarball stream through gzip\n"
119 #endif
120 " --help display this help and exit\n"
121 " --ignore-mtime use build time instead of strict per-file modification time\n"
122 " --max-extent-bytes=# set maximum decompressed extent size # in bytes\n"
123 " --preserve-mtime keep per-file modification time strictly\n"
124 " --aufs replace aufs special files with overlayfs metadata\n"
125 " --tar=[fi] generate an image from tarball(s)\n"
126 " --ovlfs-strip=[01] strip overlayfs metadata in the target image (e.g. whiteouts)\n"
127 " --quiet quiet execution (do not write anything to standard output.)\n"
128 #ifndef NDEBUG
129 " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
130 " --random-algorithms randomize per-file algorithms (debugging only)\n"
131 #endif
132 " --xattr-prefix=X X=extra xattr name prefix\n"
133 " --mount-point=X X=prefix of target fs path (default: /)\n"
134 #ifdef WITH_ANDROID
135 "\nwith following android-specific options:\n"
136 " --product-out=X X=product_out directory\n"
137 " --fs-config-file=X X=fs_config file\n"
138 " --block-list-file=X X=block_list file\n"
139 #endif
140 "\nAvailable compressors are: ", stderr);
141 print_available_compressors(stderr, ", ");
142 }
143
144 static unsigned int pclustersize_packed, pclustersize_max;
145 static struct erofs_tarfile erofstar = {
146 .global.xattrs = LIST_HEAD_INIT(erofstar.global.xattrs)
147 };
148 static bool tar_mode, rebuild_mode, gzip_supported;
149
150 static unsigned int rebuild_src_count;
151 static LIST_HEAD(rebuild_src_list);
152
parse_extended_opts(const char * opts)153 static int parse_extended_opts(const char *opts)
154 {
155 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
156 (keylen == sizeof(opt) - 1 && !memcmp(token, opt, sizeof(opt) - 1))
157
158 const char *token, *next, *tokenend, *value __maybe_unused;
159 unsigned int keylen, vallen;
160
161 value = NULL;
162 for (token = opts; *token != '\0'; token = next) {
163 bool clear = false;
164 const char *p = strchr(token, ',');
165
166 next = NULL;
167 if (p) {
168 next = p + 1;
169 } else {
170 p = token + strlen(token);
171 next = p;
172 }
173
174 tokenend = memchr(token, '=', p - token);
175 if (tokenend) {
176 keylen = tokenend - token;
177 vallen = p - tokenend - 1;
178 if (!vallen)
179 return -EINVAL;
180
181 value = tokenend + 1;
182 } else {
183 keylen = p - token;
184 vallen = 0;
185 }
186
187 if (token[0] == '^') {
188 if (keylen < 2)
189 return -EINVAL;
190 ++token;
191 --keylen;
192 clear = true;
193 }
194
195 if (MATCH_EXTENTED_OPT("legacy-compress", token, keylen)) {
196 if (vallen)
197 return -EINVAL;
198 /* disable compacted indexes and 0padding */
199 cfg.c_legacy_compress = true;
200 } else if (MATCH_EXTENTED_OPT("force-inode-compact", token, keylen)) {
201 if (vallen)
202 return -EINVAL;
203 cfg.c_force_inodeversion = FORCE_INODE_COMPACT;
204 cfg.c_ignore_mtime = true;
205 } else if (MATCH_EXTENTED_OPT("force-inode-extended", token, keylen)) {
206 if (vallen)
207 return -EINVAL;
208 cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
209 } else if (MATCH_EXTENTED_OPT("nosbcrc", token, keylen)) {
210 if (vallen)
211 return -EINVAL;
212 erofs_sb_clear_sb_chksum(&sbi);
213 } else if (MATCH_EXTENTED_OPT("noinline_data", token, keylen)) {
214 if (vallen)
215 return -EINVAL;
216 cfg.c_inline_data = false;
217 } else if (MATCH_EXTENTED_OPT("inline_data", token, keylen)) {
218 if (vallen)
219 return -EINVAL;
220 cfg.c_inline_data = !clear;
221 } else if (MATCH_EXTENTED_OPT("force-inode-blockmap", token, keylen)) {
222 if (vallen)
223 return -EINVAL;
224 cfg.c_force_chunkformat = FORCE_INODE_BLOCK_MAP;
225 } else if (MATCH_EXTENTED_OPT("force-chunk-indexes", token, keylen)) {
226 if (vallen)
227 return -EINVAL;
228 cfg.c_force_chunkformat = FORCE_INODE_CHUNK_INDEXES;
229 } else if (MATCH_EXTENTED_OPT("ztailpacking", token, keylen)) {
230 if (vallen)
231 return -EINVAL;
232 cfg.c_ztailpacking = !clear;
233 } else if (MATCH_EXTENTED_OPT("all-fragments", token, keylen)) {
234 cfg.c_all_fragments = true;
235 goto handle_fragment;
236 } else if (MATCH_EXTENTED_OPT("fragments", token, keylen)) {
237 char *endptr;
238 u64 i;
239
240 handle_fragment:
241 cfg.c_fragments = true;
242 if (vallen) {
243 i = strtoull(value, &endptr, 0);
244 if (endptr - value != vallen) {
245 erofs_err("invalid pcluster size for the packed file %s",
246 next);
247 return -EINVAL;
248 }
249 pclustersize_packed = i;
250 }
251 } else if (MATCH_EXTENTED_OPT("dedupe", token, keylen)) {
252 if (vallen)
253 return -EINVAL;
254 cfg.c_dedupe = !clear;
255 } else if (MATCH_EXTENTED_OPT("xattr-name-filter", token, keylen)) {
256 if (vallen)
257 return -EINVAL;
258 cfg.c_xattr_name_filter = !clear;
259 } else {
260 erofs_err("unknown extended option %.*s",
261 p - token, token);
262 return -EINVAL;
263 }
264 }
265 return 0;
266 }
267
mkfs_parse_compress_algs(char * algs)268 static int mkfs_parse_compress_algs(char *algs)
269 {
270 unsigned int i;
271 char *s;
272
273 for (s = strtok(algs, ":"), i = 0; s; s = strtok(NULL, ":"), ++i) {
274 const char *lv;
275
276 if (i >= EROFS_MAX_COMPR_CFGS - 1) {
277 erofs_err("too many algorithm types");
278 return -EINVAL;
279 }
280
281 lv = strchr(s, ',');
282 if (lv) {
283 cfg.c_compr_level[i] = atoi(lv + 1);
284 cfg.c_compr_alg[i] = strndup(s, lv - s);
285 } else {
286 cfg.c_compr_level[i] = -1;
287 cfg.c_compr_alg[i] = strdup(s);
288 }
289 }
290 return 0;
291 }
292
erofs_rebuild_cleanup(void)293 static void erofs_rebuild_cleanup(void)
294 {
295 struct erofs_sb_info *src, *n;
296
297 list_for_each_entry_safe(src, n, &rebuild_src_list, list) {
298 list_del(&src->list);
299 erofs_put_super(src);
300 dev_close(src);
301 free(src);
302 }
303 rebuild_src_count = 0;
304 }
305
mkfs_parse_options_cfg(int argc,char * argv[])306 static int mkfs_parse_options_cfg(int argc, char *argv[])
307 {
308 char *endptr;
309 int opt, i, err;
310 bool quiet = false;
311
312 while ((opt = getopt_long(argc, argv, "C:E:L:T:U:b:d:x:z:",
313 long_options, NULL)) != -1) {
314 switch (opt) {
315 case 'z':
316 i = mkfs_parse_compress_algs(optarg);
317 if (i)
318 return i;
319 break;
320
321 case 'b':
322 i = atoi(optarg);
323 if (i < 512 || i > EROFS_MAX_BLOCK_SIZE) {
324 erofs_err("invalid block size %s", optarg);
325 return -EINVAL;
326 }
327 sbi.blkszbits = ilog2(i);
328 break;
329
330 case 'd':
331 i = atoi(optarg);
332 if (i < EROFS_MSG_MIN || i > EROFS_MSG_MAX) {
333 erofs_err("invalid debug level %d", i);
334 return -EINVAL;
335 }
336 cfg.c_dbg_lvl = i;
337 break;
338
339 case 'x':
340 i = strtol(optarg, &endptr, 0);
341 if (*endptr != '\0') {
342 erofs_err("invalid xattr tolerance %s", optarg);
343 return -EINVAL;
344 }
345 cfg.c_inline_xattr_tolerance = i;
346 break;
347
348 case 'E':
349 opt = parse_extended_opts(optarg);
350 if (opt)
351 return opt;
352 break;
353
354 case 'L':
355 if (optarg == NULL ||
356 strlen(optarg) > sizeof(sbi.volume_name)) {
357 erofs_err("invalid volume label");
358 return -EINVAL;
359 }
360 strncpy(sbi.volume_name, optarg,
361 sizeof(sbi.volume_name));
362 break;
363
364 case 'T':
365 cfg.c_unix_timestamp = strtoull(optarg, &endptr, 0);
366 if (cfg.c_unix_timestamp == -1 || *endptr != '\0') {
367 erofs_err("invalid UNIX timestamp %s", optarg);
368 return -EINVAL;
369 }
370 cfg.c_timeinherit = TIMESTAMP_FIXED;
371 break;
372 case 'U':
373 if (erofs_uuid_parse(optarg, sbi.uuid)) {
374 erofs_err("invalid UUID %s", optarg);
375 return -EINVAL;
376 }
377 break;
378 case 2:
379 opt = erofs_parse_exclude_path(optarg, false);
380 if (opt) {
381 erofs_err("failed to parse exclude path: %s",
382 erofs_strerror(opt));
383 return opt;
384 }
385 break;
386 case 3:
387 opt = erofs_parse_exclude_path(optarg, true);
388 if (opt) {
389 erofs_err("failed to parse exclude regex: %s",
390 erofs_strerror(opt));
391 return opt;
392 }
393 break;
394
395 case 4:
396 opt = erofs_selabel_open(optarg);
397 if (opt && opt != -EBUSY)
398 return opt;
399 break;
400 case 5:
401 cfg.c_uid = strtoul(optarg, &endptr, 0);
402 if (cfg.c_uid == -1 || *endptr != '\0') {
403 erofs_err("invalid uid %s", optarg);
404 return -EINVAL;
405 }
406 break;
407 case 6:
408 cfg.c_gid = strtoul(optarg, &endptr, 0);
409 if (cfg.c_gid == -1 || *endptr != '\0') {
410 erofs_err("invalid gid %s", optarg);
411 return -EINVAL;
412 }
413 break;
414 case 7:
415 cfg.c_uid = cfg.c_gid = 0;
416 break;
417 #ifndef NDEBUG
418 case 8:
419 cfg.c_random_pclusterblks = true;
420 break;
421 case 18:
422 cfg.c_random_algorithms = true;
423 break;
424 #endif
425 case 9:
426 cfg.c_max_decompressed_extent_bytes =
427 strtoul(optarg, &endptr, 0);
428 if (*endptr != '\0') {
429 erofs_err("invalid maximum uncompressed extent size %s",
430 optarg);
431 return -EINVAL;
432 }
433 break;
434 case 10:
435 cfg.c_compress_hints_file = optarg;
436 break;
437 case 512:
438 cfg.mount_point = optarg;
439 /* all trailing '/' should be deleted */
440 opt = strlen(cfg.mount_point);
441 if (opt && optarg[opt - 1] == '/')
442 optarg[opt - 1] = '\0';
443 break;
444 #ifdef WITH_ANDROID
445 case 513:
446 cfg.target_out_path = optarg;
447 break;
448 case 514:
449 cfg.fs_config_file = optarg;
450 break;
451 case 515:
452 cfg.block_list_file = optarg;
453 break;
454 #endif
455 case 'C':
456 i = strtoull(optarg, &endptr, 0);
457 if (*endptr != '\0') {
458 erofs_err("invalid physical clustersize %s",
459 optarg);
460 return -EINVAL;
461 }
462 pclustersize_max = i;
463 break;
464 case 11:
465 i = strtol(optarg, &endptr, 0);
466 if (*endptr != '\0') {
467 erofs_err("invalid chunksize %s", optarg);
468 return -EINVAL;
469 }
470 cfg.c_chunkbits = ilog2(i);
471 if ((1 << cfg.c_chunkbits) != i) {
472 erofs_err("chunksize %s must be a power of two",
473 optarg);
474 return -EINVAL;
475 }
476 erofs_sb_set_chunked_file(&sbi);
477 break;
478 case 12:
479 quiet = true;
480 break;
481 case 13:
482 cfg.c_blobdev_path = optarg;
483 break;
484 case 14:
485 cfg.c_ignore_mtime = true;
486 break;
487 case 15:
488 cfg.c_ignore_mtime = false;
489 break;
490 case 16:
491 errno = 0;
492 cfg.c_uid_offset = strtoll(optarg, &endptr, 0);
493 if (errno || *endptr != '\0') {
494 erofs_err("invalid uid offset %s", optarg);
495 return -EINVAL;
496 }
497 break;
498 case 17:
499 errno = 0;
500 cfg.c_gid_offset = strtoll(optarg, &endptr, 0);
501 if (errno || *endptr != '\0') {
502 erofs_err("invalid gid offset %s", optarg);
503 return -EINVAL;
504 }
505 break;
506 case 19:
507 errno = 0;
508 opt = erofs_xattr_insert_name_prefix(optarg);
509 if (opt) {
510 erofs_err("failed to parse xattr name prefix: %s",
511 erofs_strerror(opt));
512 return opt;
513 }
514 cfg.c_extra_ea_name_prefixes = true;
515 break;
516 case 20:
517 if (optarg && (!strcmp(optarg, "i") ||
518 !strcmp(optarg, "0") || !memcmp(optarg, "0,", 2))) {
519 erofstar.index_mode = true;
520 if (!memcmp(optarg, "0,", 2))
521 erofstar.mapfile = strdup(optarg + 2);
522 }
523 tar_mode = true;
524 break;
525 case 21:
526 erofstar.aufs = true;
527 break;
528 case 516:
529 if (!optarg || !strcmp(optarg, "1"))
530 cfg.c_ovlfs_strip = true;
531 else
532 cfg.c_ovlfs_strip = false;
533 break;
534 case 517:
535 gzip_supported = true;
536 break;
537 case 1:
538 usage();
539 exit(0);
540
541 default: /* '?' */
542 return -EINVAL;
543 }
544 }
545
546 if (cfg.c_blobdev_path && cfg.c_chunkbits < sbi.blkszbits) {
547 erofs_err("--blobdev must be used together with --chunksize");
548 return -EINVAL;
549 }
550
551 /* TODO: can be implemented with (deviceslot) mapped_blkaddr */
552 if (cfg.c_blobdev_path &&
553 cfg.c_force_chunkformat == FORCE_INODE_BLOCK_MAP) {
554 erofs_err("--blobdev cannot work with block map currently");
555 return -EINVAL;
556 }
557
558 if (optind >= argc) {
559 erofs_err("missing argument: FILE");
560 return -EINVAL;
561 }
562
563 cfg.c_img_path = strdup(argv[optind++]);
564 if (!cfg.c_img_path)
565 return -ENOMEM;
566
567 if (optind >= argc) {
568 if (!tar_mode) {
569 erofs_err("missing argument: SOURCE(s)");
570 return -EINVAL;
571 } else {
572 int dupfd;
573
574 dupfd = dup(STDIN_FILENO);
575 if (dupfd < 0) {
576 erofs_err("failed to duplicate STDIN_FILENO: %s",
577 strerror(errno));
578 return -errno;
579 }
580 err = erofs_iostream_open(&erofstar.ios, dupfd, gzip_supported);
581 if (err)
582 return err;
583 }
584 } else {
585 struct stat st;
586
587 cfg.c_src_path = realpath(argv[optind++], NULL);
588 if (!cfg.c_src_path) {
589 erofs_err("failed to parse source directory: %s",
590 erofs_strerror(-errno));
591 return -ENOENT;
592 }
593
594 if (tar_mode) {
595 int fd = open(cfg.c_src_path, O_RDONLY);
596
597 if (fd < 0) {
598 erofs_err("failed to open file: %s", cfg.c_src_path);
599 return -errno;
600 }
601 err = erofs_iostream_open(&erofstar.ios, fd, gzip_supported);
602 if (err)
603 return err;
604 } else {
605 err = lstat(cfg.c_src_path, &st);
606 if (err)
607 return -errno;
608 if (S_ISDIR(st.st_mode))
609 erofs_set_fs_root(cfg.c_src_path);
610 else
611 rebuild_mode = true;
612 }
613
614 if (rebuild_mode) {
615 char *srcpath = cfg.c_src_path;
616 struct erofs_sb_info *src;
617
618 do {
619 src = calloc(1, sizeof(struct erofs_sb_info));
620 if (!src) {
621 erofs_rebuild_cleanup();
622 return -ENOMEM;
623 }
624
625 err = dev_open_ro(src, srcpath);
626 if (err) {
627 free(src);
628 erofs_rebuild_cleanup();
629 return err;
630 }
631
632 /* extra device index starts from 1 */
633 src->dev = ++rebuild_src_count;
634 list_add(&src->list, &rebuild_src_list);
635 } while (optind < argc && (srcpath = argv[optind++]));
636 } else if (optind < argc) {
637 erofs_err("unexpected argument: %s\n", argv[optind]);
638 return -EINVAL;
639 }
640 }
641 if (quiet) {
642 cfg.c_dbg_lvl = EROFS_ERR;
643 cfg.c_showprogress = false;
644 }
645
646 if (cfg.c_compr_alg[0] && erofs_blksiz(&sbi) != getpagesize())
647 erofs_warn("Please note that subpage blocksize with compression isn't yet supported in kernel. "
648 "This compressed image will only work with bs = ps = %u bytes",
649 erofs_blksiz(&sbi));
650
651 if (pclustersize_max) {
652 if (pclustersize_max < erofs_blksiz(&sbi) ||
653 pclustersize_max % erofs_blksiz(&sbi)) {
654 erofs_err("invalid physical clustersize %u",
655 pclustersize_max);
656 return -EINVAL;
657 }
658 cfg.c_pclusterblks_max = pclustersize_max >> sbi.blkszbits;
659 cfg.c_pclusterblks_def = cfg.c_pclusterblks_max;
660 }
661 if (cfg.c_chunkbits && cfg.c_chunkbits < sbi.blkszbits) {
662 erofs_err("chunksize %u must be larger than block size",
663 1u << cfg.c_chunkbits);
664 return -EINVAL;
665 }
666
667 if (pclustersize_packed) {
668 if (pclustersize_max < erofs_blksiz(&sbi) ||
669 pclustersize_max % erofs_blksiz(&sbi)) {
670 erofs_err("invalid pcluster size for the packed file %u",
671 pclustersize_packed);
672 return -EINVAL;
673 }
674 cfg.c_pclusterblks_packed = pclustersize_packed >> sbi.blkszbits;
675 }
676 return 0;
677 }
678
erofs_mkfs_update_super_block(struct erofs_buffer_head * bh,erofs_nid_t root_nid,erofs_blk_t * blocks,erofs_nid_t packed_nid)679 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
680 erofs_nid_t root_nid,
681 erofs_blk_t *blocks,
682 erofs_nid_t packed_nid)
683 {
684 struct erofs_super_block sb = {
685 .magic = cpu_to_le32(EROFS_SUPER_MAGIC_V1),
686 .blkszbits = sbi.blkszbits,
687 .inos = cpu_to_le64(sbi.inos),
688 .build_time = cpu_to_le64(sbi.build_time),
689 .build_time_nsec = cpu_to_le32(sbi.build_time_nsec),
690 .blocks = 0,
691 .meta_blkaddr = cpu_to_le32(sbi.meta_blkaddr),
692 .xattr_blkaddr = cpu_to_le32(sbi.xattr_blkaddr),
693 .xattr_prefix_count = sbi.xattr_prefix_count,
694 .xattr_prefix_start = cpu_to_le32(sbi.xattr_prefix_start),
695 .feature_incompat = cpu_to_le32(sbi.feature_incompat),
696 .feature_compat = cpu_to_le32(sbi.feature_compat &
697 ~EROFS_FEATURE_COMPAT_SB_CHKSUM),
698 .extra_devices = cpu_to_le16(sbi.extra_devices),
699 .devt_slotoff = cpu_to_le16(sbi.devt_slotoff),
700 };
701 const u32 sb_blksize = round_up(EROFS_SUPER_END, erofs_blksiz(&sbi));
702 char *buf;
703 int ret;
704
705 *blocks = erofs_mapbh(NULL);
706 sb.blocks = cpu_to_le32(*blocks);
707 sb.root_nid = cpu_to_le16(root_nid);
708 sb.packed_nid = cpu_to_le64(packed_nid);
709 memcpy(sb.uuid, sbi.uuid, sizeof(sb.uuid));
710 memcpy(sb.volume_name, sbi.volume_name, sizeof(sb.volume_name));
711
712 if (erofs_sb_has_compr_cfgs(&sbi))
713 sb.u1.available_compr_algs = cpu_to_le16(sbi.available_compr_algs);
714 else
715 sb.u1.lz4_max_distance = cpu_to_le16(sbi.lz4_max_distance);
716
717 buf = calloc(sb_blksize, 1);
718 if (!buf) {
719 erofs_err("failed to allocate memory for sb: %s",
720 erofs_strerror(-errno));
721 return -ENOMEM;
722 }
723 memcpy(buf + EROFS_SUPER_OFFSET, &sb, sizeof(sb));
724
725 ret = dev_write(&sbi, buf, erofs_btell(bh, false), EROFS_SUPER_END);
726 free(buf);
727 erofs_bdrop(bh, false);
728 return ret;
729 }
730
erofs_mkfs_superblock_csum_set(void)731 static int erofs_mkfs_superblock_csum_set(void)
732 {
733 int ret;
734 u8 buf[EROFS_MAX_BLOCK_SIZE];
735 u32 crc;
736 unsigned int len;
737 struct erofs_super_block *sb;
738
739 ret = blk_read(&sbi, 0, buf, 0, erofs_blknr(&sbi, EROFS_SUPER_END) + 1);
740 if (ret) {
741 erofs_err("failed to read superblock to set checksum: %s",
742 erofs_strerror(ret));
743 return ret;
744 }
745
746 /*
747 * skip the first 1024 bytes, to allow for the installation
748 * of x86 boot sectors and other oddities.
749 */
750 sb = (struct erofs_super_block *)(buf + EROFS_SUPER_OFFSET);
751
752 if (le32_to_cpu(sb->magic) != EROFS_SUPER_MAGIC_V1) {
753 erofs_err("internal error: not an erofs valid image");
754 return -EFAULT;
755 }
756
757 /* turn on checksum feature */
758 sb->feature_compat = cpu_to_le32(le32_to_cpu(sb->feature_compat) |
759 EROFS_FEATURE_COMPAT_SB_CHKSUM);
760 if (erofs_blksiz(&sbi) > EROFS_SUPER_OFFSET)
761 len = erofs_blksiz(&sbi) - EROFS_SUPER_OFFSET;
762 else
763 len = erofs_blksiz(&sbi);
764 crc = erofs_crc32c(~0, (u8 *)sb, len);
765
766 /* set up checksum field to erofs_super_block */
767 sb->checksum = cpu_to_le32(crc);
768
769 ret = blk_write(&sbi, buf, 0, 1);
770 if (ret) {
771 erofs_err("failed to write checksummed superblock: %s",
772 erofs_strerror(ret));
773 return ret;
774 }
775
776 erofs_info("superblock checksum 0x%08x written", crc);
777 return 0;
778 }
779
erofs_mkfs_default_options(void)780 static void erofs_mkfs_default_options(void)
781 {
782 cfg.c_showprogress = true;
783 cfg.c_legacy_compress = false;
784 cfg.c_inline_data = true;
785 cfg.c_xattr_name_filter = true;
786 sbi.blkszbits = ilog2(min_t(u32, getpagesize(), EROFS_MAX_BLOCK_SIZE));
787 sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_ZERO_PADDING;
788 sbi.feature_compat = EROFS_FEATURE_COMPAT_SB_CHKSUM |
789 EROFS_FEATURE_COMPAT_MTIME;
790
791 /* generate a default uuid first */
792 erofs_uuid_generate(sbi.uuid);
793 }
794
795 /* https://reproducible-builds.org/specs/source-date-epoch/ for more details */
parse_source_date_epoch(void)796 int parse_source_date_epoch(void)
797 {
798 char *source_date_epoch;
799 unsigned long long epoch = -1ULL;
800 char *endptr;
801
802 source_date_epoch = getenv("SOURCE_DATE_EPOCH");
803 if (!source_date_epoch)
804 return 0;
805
806 epoch = strtoull(source_date_epoch, &endptr, 10);
807 if (epoch == -1ULL || *endptr != '\0') {
808 erofs_err("environment variable $SOURCE_DATE_EPOCH %s is invalid",
809 source_date_epoch);
810 return -EINVAL;
811 }
812
813 if (cfg.c_force_inodeversion != FORCE_INODE_EXTENDED)
814 erofs_info("SOURCE_DATE_EPOCH is set, forcely generate extended inodes instead");
815
816 cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
817 cfg.c_unix_timestamp = epoch;
818 cfg.c_timeinherit = TIMESTAMP_CLAMPING;
819 return 0;
820 }
821
erofs_show_progs(int argc,char * argv[])822 void erofs_show_progs(int argc, char *argv[])
823 {
824 if (cfg.c_dbg_lvl >= EROFS_WARN)
825 printf("%s %s\n", basename(argv[0]), cfg.c_version);
826 }
erofs_alloc_root_inode(void)827 static struct erofs_inode *erofs_alloc_root_inode(void)
828 {
829 struct erofs_inode *root;
830
831 root = erofs_new_inode();
832 if (IS_ERR(root))
833 return root;
834 root->i_srcpath = strdup("/");
835 root->i_mode = S_IFDIR | 0777;
836 root->i_parent = root;
837 root->i_mtime = root->sbi->build_time;
838 root->i_mtime_nsec = root->sbi->build_time_nsec;
839 erofs_init_empty_dir(root);
840 return root;
841 }
842
erofs_rebuild_load_trees(struct erofs_inode * root)843 static int erofs_rebuild_load_trees(struct erofs_inode *root)
844 {
845 struct erofs_sb_info *src;
846 unsigned int extra_devices = 0;
847 erofs_blk_t nblocks;
848 int ret, idx;
849
850 list_for_each_entry(src, &rebuild_src_list, list) {
851 ret = erofs_rebuild_load_tree(root, src);
852 if (ret) {
853 erofs_err("failed to load %s", src->devname);
854 return ret;
855 }
856 if (src->extra_devices > 1) {
857 erofs_err("%s: unsupported number of extra devices",
858 src->devname, src->extra_devices);
859 return -EOPNOTSUPP;
860 }
861 extra_devices += src->extra_devices;
862 }
863
864 if (extra_devices && extra_devices != rebuild_src_count) {
865 erofs_err("extra_devices(%u) is mismatched with source images(%u)",
866 extra_devices, rebuild_src_count);
867 return -EOPNOTSUPP;
868 }
869
870 ret = erofs_mkfs_init_devices(&sbi, rebuild_src_count);
871 if (ret)
872 return ret;
873
874 list_for_each_entry(src, &rebuild_src_list, list) {
875 u8 *tag = NULL;
876
877 if (extra_devices) {
878 nblocks = src->devs[0].blocks;
879 tag = src->devs[0].tag;
880 } else {
881 nblocks = src->primarydevice_blocks;
882 }
883 DBG_BUGON(src->dev < 1);
884 idx = src->dev - 1;
885 sbi.devs[idx].blocks = nblocks;
886 if (tag && *tag)
887 memcpy(sbi.devs[idx].tag, tag, sizeof(sbi.devs[0].tag));
888 else
889 /* convert UUID of the source image to a hex string */
890 sprintf((char *)sbi.devs[idx].tag,
891 "%04x%04x%04x%04x%04x%04x%04x%04x",
892 (src->uuid[0] << 8) | src->uuid[1],
893 (src->uuid[2] << 8) | src->uuid[3],
894 (src->uuid[4] << 8) | src->uuid[5],
895 (src->uuid[6] << 8) | src->uuid[7],
896 (src->uuid[8] << 8) | src->uuid[9],
897 (src->uuid[10] << 8) | src->uuid[11],
898 (src->uuid[12] << 8) | src->uuid[13],
899 (src->uuid[14] << 8) | src->uuid[15]);
900 }
901 return 0;
902 }
903
erofs_mkfs_showsummaries(erofs_blk_t nblocks)904 static void erofs_mkfs_showsummaries(erofs_blk_t nblocks)
905 {
906 char uuid_str[37] = {};
907
908 if (!(cfg.c_dbg_lvl > EROFS_ERR && cfg.c_showprogress))
909 return;
910
911 erofs_uuid_unparse_lower(sbi.uuid, uuid_str);
912
913 fprintf(stdout, "------\nFilesystem UUID: %s\n"
914 "Filesystem total blocks: %u (of %u-byte blocks)\n"
915 "Filesystem total inodes: %llu\n"
916 "Filesystem total metadata blocks: %u\n"
917 "Filesystem total deduplicated bytes (of source files): %llu\n",
918 uuid_str, nblocks, 1U << sbi.blkszbits, sbi.inos | 0ULL,
919 erofs_total_metablocks(),
920 sbi.saved_by_deduplication | 0ULL);
921 }
922
main(int argc,char ** argv)923 int main(int argc, char **argv)
924 {
925 int err = 0;
926 struct erofs_buffer_head *sb_bh;
927 struct erofs_inode *root_inode, *packed_inode;
928 erofs_nid_t root_nid, packed_nid;
929 erofs_blk_t nblocks;
930 struct timeval t;
931 FILE *packedfile = NULL;
932
933 erofs_init_configure();
934 erofs_mkfs_default_options();
935
936 err = mkfs_parse_options_cfg(argc, argv);
937 erofs_show_progs(argc, argv);
938 if (err) {
939 if (err == -EINVAL)
940 usage();
941 return 1;
942 }
943
944 err = parse_source_date_epoch();
945 if (err) {
946 usage();
947 return 1;
948 }
949
950 if (cfg.c_unix_timestamp != -1) {
951 sbi.build_time = cfg.c_unix_timestamp;
952 sbi.build_time_nsec = 0;
953 } else if (!gettimeofday(&t, NULL)) {
954 sbi.build_time = t.tv_sec;
955 sbi.build_time_nsec = t.tv_usec;
956 }
957
958 err = dev_open(&sbi, cfg.c_img_path);
959 if (err) {
960 usage();
961 return 1;
962 }
963
964 if (tar_mode && !erofstar.index_mode) {
965 err = erofs_diskbuf_init(1);
966 if (err) {
967 erofs_err("failed to initialize diskbuf: %s",
968 strerror(-err));
969 goto exit;
970 }
971 }
972 #ifdef WITH_ANDROID
973 if (cfg.fs_config_file &&
974 load_canned_fs_config(cfg.fs_config_file) < 0) {
975 erofs_err("failed to load fs config %s", cfg.fs_config_file);
976 return 1;
977 }
978
979 if (cfg.block_list_file &&
980 erofs_blocklist_open(cfg.block_list_file, false)) {
981 erofs_err("failed to open %s", cfg.block_list_file);
982 return 1;
983 }
984 #endif
985 erofs_show_config();
986 if (cfg.c_fragments || cfg.c_extra_ea_name_prefixes) {
987 if (!cfg.c_pclusterblks_packed)
988 cfg.c_pclusterblks_packed = cfg.c_pclusterblks_def;
989
990 packedfile = erofs_packedfile_init();
991 if (IS_ERR(packedfile)) {
992 erofs_err("failed to initialize packedfile");
993 return 1;
994 }
995 }
996
997 if (cfg.c_fragments) {
998 err = z_erofs_fragments_init();
999 if (err) {
1000 erofs_err("failed to initialize fragments");
1001 return 1;
1002 }
1003 }
1004
1005 #ifndef NDEBUG
1006 if (cfg.c_random_pclusterblks)
1007 srand(time(NULL));
1008 #endif
1009 if (tar_mode && erofstar.index_mode) {
1010 if (erofstar.mapfile) {
1011 err = erofs_blocklist_open(erofstar.mapfile, true);
1012 if (err) {
1013 erofs_err("failed to open %s", erofstar.mapfile);
1014 goto exit;
1015 }
1016 } else {
1017 sbi.blkszbits = 9;
1018 }
1019 }
1020
1021 if (rebuild_mode) {
1022 struct erofs_sb_info *src;
1023
1024 erofs_warn("EXPERIMENTAL rebuild mode in use. Use at your own risk!");
1025
1026 src = list_first_entry(&rebuild_src_list, struct erofs_sb_info, list);
1027 if (!src)
1028 goto exit;
1029 err = erofs_read_superblock(src);
1030 if (err) {
1031 erofs_err("failed to read superblock of %s", src->devname);
1032 goto exit;
1033 }
1034 sbi.blkszbits = src->blkszbits;
1035 }
1036
1037 sb_bh = erofs_buffer_init();
1038 if (IS_ERR(sb_bh)) {
1039 err = PTR_ERR(sb_bh);
1040 erofs_err("failed to initialize buffers: %s",
1041 erofs_strerror(err));
1042 goto exit;
1043 }
1044 err = erofs_bh_balloon(sb_bh, EROFS_SUPER_END);
1045 if (err < 0) {
1046 erofs_err("failed to balloon erofs_super_block: %s",
1047 erofs_strerror(err));
1048 goto exit;
1049 }
1050
1051 /* make sure that the super block should be the very first blocks */
1052 (void)erofs_mapbh(sb_bh->block);
1053 if (erofs_btell(sb_bh, false) != 0) {
1054 erofs_err("failed to reserve erofs_super_block");
1055 goto exit;
1056 }
1057
1058 err = erofs_load_compress_hints(&sbi);
1059 if (err) {
1060 erofs_err("failed to load compress hints %s",
1061 cfg.c_compress_hints_file);
1062 goto exit;
1063 }
1064
1065 err = z_erofs_compress_init(&sbi, sb_bh);
1066 if (err) {
1067 erofs_err("failed to initialize compressor: %s",
1068 erofs_strerror(err));
1069 goto exit;
1070 }
1071
1072 if (cfg.c_dedupe) {
1073 if (!cfg.c_compr_alg[0]) {
1074 erofs_err("Compression is not enabled. Turn on chunk-based data deduplication instead.");
1075 cfg.c_chunkbits = sbi.blkszbits;
1076 } else {
1077 err = z_erofs_dedupe_init(erofs_blksiz(&sbi));
1078 if (err) {
1079 erofs_err("failed to initialize deduplication: %s",
1080 erofs_strerror(err));
1081 goto exit;
1082 }
1083 }
1084 }
1085
1086 if (cfg.c_chunkbits) {
1087 err = erofs_blob_init(cfg.c_blobdev_path, 1 << cfg.c_chunkbits);
1088 if (err)
1089 return 1;
1090 }
1091
1092 if ((erofstar.index_mode && !erofstar.mapfile) || cfg.c_blobdev_path)
1093 err = erofs_mkfs_init_devices(&sbi, 1);
1094 if (err) {
1095 erofs_err("failed to generate device table: %s",
1096 erofs_strerror(err));
1097 goto exit;
1098 }
1099
1100 erofs_inode_manager_init();
1101
1102 if (tar_mode) {
1103 root_inode = erofs_alloc_root_inode();
1104 if (IS_ERR(root_inode)) {
1105 err = PTR_ERR(root_inode);
1106 goto exit;
1107 }
1108
1109 while (!(err = tarerofs_parse_tar(root_inode, &erofstar)));
1110
1111 if (err < 0)
1112 goto exit;
1113
1114 err = erofs_rebuild_dump_tree(root_inode);
1115 if (err < 0)
1116 goto exit;
1117 } else if (rebuild_mode) {
1118 root_inode = erofs_alloc_root_inode();
1119 if (IS_ERR(root_inode)) {
1120 err = PTR_ERR(root_inode);
1121 goto exit;
1122 }
1123
1124 err = erofs_rebuild_load_trees(root_inode);
1125 if (err)
1126 goto exit;
1127 err = erofs_rebuild_dump_tree(root_inode);
1128 if (err)
1129 goto exit;
1130 } else {
1131 err = erofs_build_shared_xattrs_from_path(&sbi, cfg.c_src_path);
1132 if (err) {
1133 erofs_err("failed to build shared xattrs: %s",
1134 erofs_strerror(err));
1135 goto exit;
1136 }
1137
1138 if (cfg.c_extra_ea_name_prefixes)
1139 erofs_xattr_write_name_prefixes(&sbi, packedfile);
1140
1141 root_inode = erofs_mkfs_build_tree_from_path(cfg.c_src_path);
1142 if (IS_ERR(root_inode)) {
1143 err = PTR_ERR(root_inode);
1144 goto exit;
1145 }
1146 }
1147 root_nid = erofs_lookupnid(root_inode);
1148 erofs_iput(root_inode);
1149
1150 if (erofstar.index_mode || cfg.c_chunkbits || sbi.extra_devices) {
1151 if (erofstar.index_mode && !erofstar.mapfile)
1152 sbi.devs[0].blocks =
1153 BLK_ROUND_UP(&sbi, erofstar.offset);
1154 err = erofs_mkfs_dump_blobs(&sbi);
1155 if (err)
1156 goto exit;
1157 }
1158
1159 packed_nid = 0;
1160 if ((cfg.c_fragments || cfg.c_extra_ea_name_prefixes) &&
1161 erofs_sb_has_fragments(&sbi)) {
1162 erofs_update_progressinfo("Handling packed_file ...");
1163 packed_inode = erofs_mkfs_build_packedfile();
1164 if (IS_ERR(packed_inode)) {
1165 err = PTR_ERR(packed_inode);
1166 goto exit;
1167 }
1168 packed_nid = erofs_lookupnid(packed_inode);
1169 erofs_iput(packed_inode);
1170 }
1171
1172 /* flush all buffers except for the superblock */
1173 if (!erofs_bflush(NULL)) {
1174 err = -EIO;
1175 goto exit;
1176 }
1177
1178 err = erofs_mkfs_update_super_block(sb_bh, root_nid, &nblocks,
1179 packed_nid);
1180 if (err)
1181 goto exit;
1182
1183 /* flush all remaining buffers */
1184 if (!erofs_bflush(NULL))
1185 err = -EIO;
1186 else
1187 err = dev_resize(&sbi, nblocks);
1188
1189 if (!err && erofs_sb_has_sb_chksum(&sbi))
1190 err = erofs_mkfs_superblock_csum_set();
1191 exit:
1192 z_erofs_compress_exit();
1193 z_erofs_dedupe_exit();
1194 erofs_blocklist_close();
1195 dev_close(&sbi);
1196 erofs_cleanup_compress_hints();
1197 erofs_cleanup_exclude_rules();
1198 if (cfg.c_chunkbits)
1199 erofs_blob_exit();
1200 if (cfg.c_fragments)
1201 z_erofs_fragments_exit();
1202 erofs_packedfile_exit();
1203 erofs_xattr_cleanup_name_prefixes();
1204 erofs_rebuild_cleanup();
1205 erofs_diskbuf_exit();
1206 erofs_exit_configure();
1207 if (tar_mode)
1208 erofs_iostream_close(&erofstar.ios);
1209
1210 if (err) {
1211 erofs_err("\tCould not format the device : %s\n",
1212 erofs_strerror(err));
1213 return 1;
1214 }
1215 erofs_update_progressinfo("Build completed.\n");
1216 erofs_mkfs_showsummaries(nblocks);
1217 return 0;
1218 }
1219