1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 *******************************************************************************
5 *
6 * Copyright (C) 2005-2014, International Business Machines
7 * Corporation and others. All Rights Reserved.
8 *
9 *******************************************************************************
10 * file name: icupkg.cpp
11 * encoding: UTF-8
12 * tab size: 8 (not used)
13 * indentation:4
14 *
15 * created on: 2005jul29
16 * created by: Markus W. Scherer
17 *
18 * This tool operates on ICU data (.dat package) files.
19 * It takes one as input, or creates an empty one, and can remove, add, and
20 * extract data pieces according to command-line options.
21 * At the same time, it swaps each piece to a consistent set of platform
22 * properties as desired.
23 * Useful as an install-time tool for shipping only one flavor of ICU data
24 * and preparing data files for the target platform.
25 * Also for customizing ICU data (pruning, augmenting, replacing) and for
26 * taking it apart.
27 * Subsumes functionality and implementation code from
28 * gencmn, decmn, and icuswap tools.
29 * Will not work with data DLLs (shared libraries).
30 */
31
32 #include "unicode/utypes.h"
33 #include "unicode/putil.h"
34 #include "cstring.h"
35 #include "toolutil.h"
36 #include "uoptions.h"
37 #include "uparse.h"
38 #include "filestrm.h"
39 #include "package.h"
40 #include "pkg_icu.h"
41
42 #include <stdio.h>
43 #include <stdlib.h>
44 #include <string.h>
45
46 U_NAMESPACE_USE
47
48 // TODO: add --matchmode=regex for using the ICU regex engine for item name pattern matching?
49
50 // general definitions ----------------------------------------------------- ***
51
52 // main() ------------------------------------------------------------------ ***
53
54 static void
printUsage(const char * pname,UBool isHelp)55 printUsage(const char *pname, UBool isHelp) {
56 FILE *where=isHelp ? stdout : stderr;
57
58 fprintf(where,
59 "%csage: %s [-h|-?|--help ] [-tl|-tb|-te] [-c] [-C comment]\n"
60 "\t[-a list] [-r list] [-x list] [-l [-o outputListFileName]]\n"
61 "\t[-s path] [-d path] [-w] [-m mode]\n"
62 "\t[--auto_toc_prefix] [--auto_toc_prefix_with_type] [--toc_prefix]\n"
63 "\tinfilename [outfilename]\n",
64 isHelp ? 'U' : 'u', pname);
65 if(isHelp) {
66 fprintf(where,
67 "\n"
68 "Read the input ICU .dat package file, modify it according to the options,\n"
69 "swap it to the desired platform properties (charset & endianness),\n"
70 "and optionally write the resulting ICU .dat package to the output file.\n"
71 "Items are removed, then added, then extracted and listed.\n"
72 "An ICU .dat package is written if items are removed or added,\n"
73 "or if the input and output filenames differ,\n"
74 "or if the --writepkg (-w) option is set.\n");
75 fprintf(where,
76 "\n"
77 "If the input filename is \"new\" then an empty package is created.\n"
78 "If the output filename is missing, then it is automatically generated\n"
79 "from the input filename: If the input filename ends with an l, b, or e\n"
80 "matching its platform properties, then the output filename will\n"
81 "contain the letter from the -t (--type) option.\n");
82 fprintf(where,
83 "\n"
84 "This tool can also be used to just swap a single ICU data file, replacing the\n"
85 "former icuswap tool. For this mode, provide the infilename (and optional\n"
86 "outfilename) for a non-package ICU data file.\n"
87 "Allowed options include -t, -w, -s and -d.\n"
88 "The filenames can be absolute, or relative to the source/dest dir paths.\n"
89 "Other options are not allowed in this mode.\n");
90 fprintf(where,
91 "\n"
92 "Options:\n"
93 "\t(Only the last occurrence of an option is used.)\n"
94 "\n"
95 "\t-h or -? or --help print this message and exit\n");
96 fprintf(where,
97 "\n"
98 "\t-tl or --type l output for little-endian/ASCII charset family\n"
99 "\t-tb or --type b output for big-endian/ASCII charset family\n"
100 "\t-te or --type e output for big-endian/EBCDIC charset family\n"
101 "\t The output type defaults to the input type.\n"
102 "\n"
103 "\t-c or --copyright include the ICU copyright notice\n"
104 "\t-C comment or --comment comment include a comment string\n");
105 fprintf(where,
106 "\n"
107 "\t-a list or --add list add items to the package\n"
108 "\t-r list or --remove list remove items from the package\n"
109 "\t-x list or --extract list extract items from the package\n"
110 "\tThe list can be a single item's filename,\n"
111 "\tor a .txt filename with a list of item filenames,\n"
112 "\tor an ICU .dat package filename.\n");
113 fprintf(where,
114 "\n"
115 "\t-w or --writepkg write the output package even if no items are removed\n"
116 "\t or added (e.g., for only swapping the data)\n");
117 fprintf(where,
118 "\n"
119 "\t-m mode or --matchmode mode set the matching mode for item names with\n"
120 "\t wildcards\n"
121 "\t noslash: the '*' wildcard does not match the '/' tree separator\n");
122 fprintf(where,
123 "\n"
124 "\tIn the .dat package, the Table of Contents (ToC) contains an entry\n"
125 "\tfor each item of the form prefix/tree/itemname .\n"
126 "\tThe prefix normally matches the package basename, and icupkg checks that,\n"
127 "\tbut this is not necessary when ICU need not find and load the package by filename.\n"
128 "\tICU package names end with the platform type letter, and thus differ\n"
129 "\tbetween platform types. This is not required for user data packages.\n");
130 fprintf(where,
131 "\n"
132 "\t--auto_toc_prefix automatic ToC entries prefix\n"
133 "\t Uses the prefix of the first entry of the\n"
134 "\t input package, rather than its basename.\n"
135 "\t Requires a non-empty input package.\n"
136 "\t--auto_toc_prefix_with_type auto_toc_prefix + adjust platform type\n"
137 "\t Same as auto_toc_prefix but also checks that\n"
138 "\t the prefix ends with the input platform\n"
139 "\t type letter, and modifies it to the output\n"
140 "\t platform type letter.\n"
141 "\t At most one of the auto_toc_prefix options\n"
142 "\t can be used at a time.\n"
143 "\t--toc_prefix prefix ToC prefix to be used in the output package\n"
144 "\t Overrides the package basename\n"
145 "\t and --auto_toc_prefix.\n"
146 "\t Cannot be combined with --auto_toc_prefix_with_type.\n");
147 /*
148 * Usage text columns, starting after the initial TAB.
149 * 1 2 3 4 5 6 7 8
150 * 901234567890123456789012345678901234567890123456789012345678901234567890
151 */
152 fprintf(where,
153 "\n"
154 "\tList file syntax: Items are listed on one or more lines and separated\n"
155 "\tby whitespace (space+tab).\n"
156 "\tComments begin with # and are ignored. Empty lines are ignored.\n"
157 "\tLines where the first non-whitespace character is one of %s\n"
158 "\tare also ignored, to reserve for future syntax.\n",
159 U_PKG_RESERVED_CHARS);
160 fprintf(where,
161 "\tItems for removal or extraction may contain a single '*' wildcard\n"
162 "\tcharacter. The '*' matches zero or more characters.\n"
163 "\tIf --matchmode noslash (-m noslash) is set, then the '*'\n"
164 "\tdoes not match '/'.\n");
165 fprintf(where,
166 "\n"
167 "\tItems must be listed relative to the package, and the --sourcedir or\n"
168 "\tthe --destdir path will be prepended.\n"
169 "\tThe paths are only prepended to item filenames while adding or\n"
170 "\textracting items, not to ICU .dat package or list filenames.\n"
171 "\t\n"
172 "\tPaths may contain '/' instead of the platform's\n"
173 "\tfile separator character, and are converted as appropriate.\n");
174 fprintf(where,
175 "\n"
176 "\t-s path or --sourcedir path directory for the --add items\n"
177 "\t-d path or --destdir path directory for the --extract items\n"
178 "\n"
179 "\t-l or --list list the package items\n"
180 "\t (after modifying the package)\n"
181 "\t to stdout or to output list file\n"
182 "\t-o path or --outlist path path/filename for the --list output\n");
183 }
184 }
185
186 static UOption options[]={
187 UOPTION_HELP_H,
188 UOPTION_HELP_QUESTION_MARK,
189 UOPTION_DEF("type", 't', UOPT_REQUIRES_ARG),
190
191 UOPTION_COPYRIGHT,
192 UOPTION_DEF("comment", 'C', UOPT_REQUIRES_ARG),
193
194 UOPTION_SOURCEDIR,
195 UOPTION_DESTDIR,
196
197 UOPTION_DEF("writepkg", 'w', UOPT_NO_ARG),
198
199 UOPTION_DEF("matchmode", 'm', UOPT_REQUIRES_ARG),
200
201 UOPTION_DEF("add", 'a', UOPT_REQUIRES_ARG),
202 UOPTION_DEF("remove", 'r', UOPT_REQUIRES_ARG),
203 UOPTION_DEF("extract", 'x', UOPT_REQUIRES_ARG),
204
205 UOPTION_DEF("list", 'l', UOPT_NO_ARG),
206 UOPTION_DEF("outlist", 'o', UOPT_REQUIRES_ARG),
207
208 UOPTION_DEF("auto_toc_prefix", '\1', UOPT_NO_ARG),
209 UOPTION_DEF("auto_toc_prefix_with_type", '\1', UOPT_NO_ARG),
210 UOPTION_DEF("toc_prefix", '\1', UOPT_REQUIRES_ARG)
211 };
212
213 enum {
214 OPT_HELP_H,
215 OPT_HELP_QUESTION_MARK,
216 OPT_OUT_TYPE,
217
218 OPT_COPYRIGHT,
219 OPT_COMMENT,
220
221 OPT_SOURCEDIR,
222 OPT_DESTDIR,
223
224 OPT_WRITEPKG,
225
226 OPT_MATCHMODE,
227
228 OPT_ADD_LIST,
229 OPT_REMOVE_LIST,
230 OPT_EXTRACT_LIST,
231
232 OPT_LIST_ITEMS,
233 OPT_LIST_FILE,
234
235 OPT_AUTO_TOC_PREFIX,
236 OPT_AUTO_TOC_PREFIX_WITH_TYPE,
237 OPT_TOC_PREFIX,
238
239 OPT_COUNT
240 };
241
242 static UBool
isPackageName(const char * filename)243 isPackageName(const char *filename) {
244 int32_t len;
245
246 len=(int32_t)strlen(filename)-4; /* -4: subtract the length of ".dat" */
247 return (UBool)(len>0 && 0==strcmp(filename+len, ".dat"));
248 }
249 /*
250 This line is required by MinGW because it incorrectly globs the arguments.
251 So when \* is used, it turns into a list of files instead of a literal "*"
252 */
253 int _CRT_glob = 0;
254
255 extern int
main(int argc,char * argv[])256 main(int argc, char *argv[]) {
257 const char *pname, *sourcePath, *destPath, *inFilename, *outFilename, *outComment;
258 char outType;
259 UBool isHelp, isModified, isPackage;
260 int result = 0;
261
262 Package *pkg, *listPkg, *addListPkg;
263
264 U_MAIN_INIT_ARGS(argc, argv);
265
266 /* get the program basename */
267 pname=findBasename(argv[0]);
268
269 argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
270 isHelp=options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur;
271 if(isHelp) {
272 printUsage(pname, TRUE);
273 return U_ZERO_ERROR;
274 }
275
276 pkg=new Package;
277 if(pkg==NULL) {
278 fprintf(stderr, "icupkg: not enough memory\n");
279 return U_MEMORY_ALLOCATION_ERROR;
280 }
281 isModified=FALSE;
282
283 int autoPrefix=0;
284 if(options[OPT_AUTO_TOC_PREFIX].doesOccur) {
285 pkg->setAutoPrefix();
286 ++autoPrefix;
287 }
288 if(options[OPT_AUTO_TOC_PREFIX_WITH_TYPE].doesOccur) {
289 if(options[OPT_TOC_PREFIX].doesOccur) {
290 fprintf(stderr, "icupkg: --auto_toc_prefix_with_type and also --toc_prefix\n");
291 printUsage(pname, FALSE);
292 return U_ILLEGAL_ARGUMENT_ERROR;
293 }
294 pkg->setAutoPrefixWithType();
295 ++autoPrefix;
296 }
297 if(argc<2 || 3<argc || autoPrefix>1) {
298 printUsage(pname, FALSE);
299 return U_ILLEGAL_ARGUMENT_ERROR;
300 }
301
302 if(options[OPT_SOURCEDIR].doesOccur) {
303 sourcePath=options[OPT_SOURCEDIR].value;
304 } else {
305 // work relative to the current working directory
306 sourcePath=NULL;
307 }
308 if(options[OPT_DESTDIR].doesOccur) {
309 destPath=options[OPT_DESTDIR].value;
310 } else {
311 // work relative to the current working directory
312 destPath=NULL;
313 }
314
315 if(0==strcmp(argv[1], "new")) {
316 if(autoPrefix) {
317 fprintf(stderr, "icupkg: --auto_toc_prefix[_with_type] but no input package\n");
318 printUsage(pname, FALSE);
319 return U_ILLEGAL_ARGUMENT_ERROR;
320 }
321 inFilename=NULL;
322 isPackage=TRUE;
323 } else {
324 inFilename=argv[1];
325 if(isPackageName(inFilename)) {
326 pkg->readPackage(inFilename);
327 isPackage=TRUE;
328 } else {
329 /* swap a single file (icuswap replacement) rather than work on a package */
330 pkg->addFile(sourcePath, inFilename);
331 isPackage=FALSE;
332 }
333 }
334
335 if(argc>=3) {
336 outFilename=argv[2];
337 if(0!=strcmp(argv[1], argv[2])) {
338 isModified=TRUE;
339 }
340 } else if(isPackage) {
341 outFilename=NULL;
342 } else /* !isPackage */ {
343 outFilename=inFilename;
344 isModified=(UBool)(sourcePath!=destPath);
345 }
346
347 /* parse the output type option */
348 if(options[OPT_OUT_TYPE].doesOccur) {
349 const char *type=options[OPT_OUT_TYPE].value;
350 if(type[0]==0 || type[1]!=0) {
351 /* the type must be exactly one letter */
352 printUsage(pname, FALSE);
353 return U_ILLEGAL_ARGUMENT_ERROR;
354 }
355 outType=type[0];
356 switch(outType) {
357 case 'l':
358 case 'b':
359 case 'e':
360 break;
361 default:
362 printUsage(pname, FALSE);
363 return U_ILLEGAL_ARGUMENT_ERROR;
364 }
365
366 /*
367 * Set the isModified flag if the output type differs from the
368 * input package type.
369 * If we swap a single file, just assume that we are modifying it.
370 * The Package class does not give us access to the item and its type.
371 */
372 isModified|=(UBool)(!isPackage || outType!=pkg->getInType());
373 } else if(isPackage) {
374 outType=pkg->getInType(); // default to input type
375 } else /* !isPackage: swap single file */ {
376 outType=0; /* tells extractItem() to not swap */
377 }
378
379 if(options[OPT_WRITEPKG].doesOccur) {
380 isModified=TRUE;
381 }
382
383 if(!isPackage) {
384 /*
385 * icuswap tool replacement: Only swap a single file.
386 * Check that irrelevant options are not set.
387 */
388 if( options[OPT_COMMENT].doesOccur ||
389 options[OPT_COPYRIGHT].doesOccur ||
390 options[OPT_MATCHMODE].doesOccur ||
391 options[OPT_REMOVE_LIST].doesOccur ||
392 options[OPT_ADD_LIST].doesOccur ||
393 options[OPT_EXTRACT_LIST].doesOccur ||
394 options[OPT_LIST_ITEMS].doesOccur
395 ) {
396 printUsage(pname, FALSE);
397 return U_ILLEGAL_ARGUMENT_ERROR;
398 }
399 if(isModified) {
400 pkg->extractItem(destPath, outFilename, 0, outType);
401 }
402
403 delete pkg;
404 return result;
405 }
406
407 /* Work with a package. */
408
409 if(options[OPT_COMMENT].doesOccur) {
410 outComment=options[OPT_COMMENT].value;
411 } else if(options[OPT_COPYRIGHT].doesOccur) {
412 outComment=U_COPYRIGHT_STRING;
413 } else {
414 outComment=NULL;
415 }
416
417 if(options[OPT_MATCHMODE].doesOccur) {
418 if(0==strcmp(options[OPT_MATCHMODE].value, "noslash")) {
419 pkg->setMatchMode(Package::MATCH_NOSLASH);
420 } else {
421 printUsage(pname, FALSE);
422 return U_ILLEGAL_ARGUMENT_ERROR;
423 }
424 }
425
426 /* remove items */
427 if(options[OPT_REMOVE_LIST].doesOccur) {
428 listPkg=new Package();
429 if(listPkg==NULL) {
430 fprintf(stderr, "icupkg: not enough memory\n");
431 exit(U_MEMORY_ALLOCATION_ERROR);
432 }
433 if(readList(NULL, options[OPT_REMOVE_LIST].value, FALSE, listPkg)) {
434 pkg->removeItems(*listPkg);
435 delete listPkg;
436 isModified=TRUE;
437 } else {
438 printUsage(pname, FALSE);
439 return U_ILLEGAL_ARGUMENT_ERROR;
440 }
441 }
442
443 /*
444 * add items
445 * use a separate Package so that its memory and items stay around
446 * as long as the main Package
447 */
448 addListPkg=NULL;
449 if(options[OPT_ADD_LIST].doesOccur) {
450 addListPkg=new Package();
451 if(addListPkg==NULL) {
452 fprintf(stderr, "icupkg: not enough memory\n");
453 exit(U_MEMORY_ALLOCATION_ERROR);
454 }
455 if(readList(sourcePath, options[OPT_ADD_LIST].value, TRUE, addListPkg)) {
456 pkg->addItems(*addListPkg);
457 // delete addListPkg; deferred until after writePackage()
458 isModified=TRUE;
459 } else {
460 printUsage(pname, FALSE);
461 return U_ILLEGAL_ARGUMENT_ERROR;
462 }
463 }
464
465 /* extract items */
466 if(options[OPT_EXTRACT_LIST].doesOccur) {
467 listPkg=new Package();
468 if(listPkg==NULL) {
469 fprintf(stderr, "icupkg: not enough memory\n");
470 exit(U_MEMORY_ALLOCATION_ERROR);
471 }
472 if(readList(NULL, options[OPT_EXTRACT_LIST].value, FALSE, listPkg)) {
473 pkg->extractItems(destPath, *listPkg, outType);
474 delete listPkg;
475 } else {
476 printUsage(pname, FALSE);
477 return U_ILLEGAL_ARGUMENT_ERROR;
478 }
479 }
480
481 /* list items */
482 if(options[OPT_LIST_ITEMS].doesOccur) {
483 int32_t i;
484 if (options[OPT_LIST_FILE].doesOccur) {
485 FileStream *out;
486 out = T_FileStream_open(options[OPT_LIST_FILE].value, "w");
487 if (out != NULL) {
488 for(i=0; i<pkg->getItemCount(); ++i) {
489 T_FileStream_writeLine(out, pkg->getItem(i)->name);
490 T_FileStream_writeLine(out, "\n");
491 }
492 T_FileStream_close(out);
493 } else {
494 return U_ILLEGAL_ARGUMENT_ERROR;
495 }
496 } else {
497 for(i=0; i<pkg->getItemCount(); ++i) {
498 fprintf(stdout, "%s\n", pkg->getItem(i)->name);
499 }
500 }
501 }
502
503 /* check dependencies between items */
504 if(!pkg->checkDependencies()) {
505 /* some dependencies are not fulfilled */
506 return U_MISSING_RESOURCE_ERROR;
507 }
508
509 /* write the output .dat package if there are any modifications */
510 if(isModified) {
511 char outFilenameBuffer[1024]; // for auto-generated output filename, if necessary
512
513 if(outFilename==NULL || outFilename[0]==0) {
514 if(inFilename==NULL || inFilename[0]==0) {
515 fprintf(stderr, "icupkg: unable to auto-generate an output filename if there is no input filename\n");
516 exit(U_ILLEGAL_ARGUMENT_ERROR);
517 }
518
519 /*
520 * auto-generate a filename:
521 * copy the inFilename,
522 * and if the last basename character matches the input file's type,
523 * then replace it with the output file's type
524 */
525 char suffix[6]="?.dat";
526 char *s;
527
528 suffix[0]=pkg->getInType();
529 strcpy(outFilenameBuffer, inFilename);
530 s=strchr(outFilenameBuffer, 0);
531 if((s-outFilenameBuffer)>5 && 0==memcmp(s-5, suffix, 5)) {
532 *(s-5)=outType;
533 }
534 outFilename=outFilenameBuffer;
535 }
536 if(options[OPT_TOC_PREFIX].doesOccur) {
537 pkg->setPrefix(options[OPT_TOC_PREFIX].value);
538 }
539 result = writePackageDatFile(outFilename, outComment, NULL, NULL, pkg, outType);
540 }
541
542 delete addListPkg;
543 delete pkg;
544 return result;
545 }
546
547 /*
548 * Hey, Emacs, please set the following:
549 *
550 * Local Variables:
551 * indent-tabs-mode: nil
552 * End:
553 *
554 */
555