1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 **********************************************************************
5 * Copyright (C) 2002-2016, International Business Machines
6 * Corporation and others. All Rights Reserved.
7 **********************************************************************
8 *
9 * File gendict.cpp
10 */
11
12 #include "unicode/utypes.h"
13 #include "unicode/uchar.h"
14 #include "unicode/ucnv.h"
15 #include "unicode/uniset.h"
16 #include "unicode/unistr.h"
17 #include "unicode/uclean.h"
18 #include "unicode/udata.h"
19 #include "unicode/putil.h"
20 #include "unicode/ucharstriebuilder.h"
21 #include "unicode/bytestriebuilder.h"
22 #include "unicode/ucharstrie.h"
23 #include "unicode/bytestrie.h"
24 #include "unicode/ucnv.h"
25 #include "unicode/ustring.h"
26 #include "unicode/utf16.h"
27
28 #include "charstr.h"
29 #include "dictionarydata.h"
30 #include "uoptions.h"
31 #include "unewdata.h"
32 #include "cmemory.h"
33 #include "uassert.h"
34 #include "ucbuf.h"
35 #include "toolutil.h"
36 #include "cstring.h"
37 #include "writesrc.h"
38
39 #include <stdio.h>
40 #include <stdlib.h>
41 #include <string.h>
42
43 #include "putilimp.h"
44 UDate startTime;
45
elapsedTime()46 static int elapsedTime() {
47 return static_cast<int>(uprv_floor((uprv_getRawUTCtime() - startTime) / 1000.0));
48 }
49
50 U_NAMESPACE_USE
51
52 static char *progName;
53 static UOption options[]={
54 UOPTION_HELP_H, /* 0 */
55 UOPTION_HELP_QUESTION_MARK, /* 1 */
56 UOPTION_VERBOSE, /* 2 */
57 UOPTION_ICUDATADIR, /* 4 */
58 UOPTION_COPYRIGHT, /* 5 */
59 { "uchars", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0}, /* 6 */
60 { "bytes", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0}, /* 7 */
61 { "transform", nullptr, nullptr, nullptr, '\1', UOPT_REQUIRES_ARG, 0}, /* 8 */
62 { "toml", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0}, /* 9 */
63 UOPTION_QUIET, /* 10 */
64 };
65
66 enum arguments {
67 ARG_HELP = 0,
68 ARG_QMARK,
69 ARG_VERBOSE,
70 ARG_ICUDATADIR,
71 ARG_COPYRIGHT,
72 ARG_UCHARS,
73 ARG_BYTES,
74 ARG_TRANSFORM,
75 ARG_TOML,
76 ARG_QUIET
77 };
78
79 // prints out the standard usage method describing command line arguments,
80 // then bails out with the desired exit code
usageAndDie(UErrorCode retCode)81 static void usageAndDie(UErrorCode retCode) {
82 fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName);
83 fprintf((U_SUCCESS(retCode) ? stdout : stderr),
84 "\tRead in a word list and write out a string trie dictionary\n"
85 "options:\n"
86 "\t-h or -? or --help this usage text\n"
87 "\t-V or --version show a version message\n"
88 "\t-c or --copyright include a copyright notice\n"
89 "\t-v or --verbose turn on verbose output\n"
90 "\t-q or --quiet do not display warnings and progress\n"
91 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option
92 "\t followed by path, defaults to %s\n"
93 "\t--uchars output a UCharsTrie (mutually exclusive with -b!)\n"
94 "\t--bytes output a BytesTrie (mutually exclusive with -u!)\n"
95 "\t--transform the kind of transform to use (eg --transform offset-40A3,\n"
96 "\t which specifies an offset transform with constant 0x40A3)\n"
97 "\t--toml output the trie in toml format (default is binary),\n",
98 u_getDataDirectory());
99 exit(retCode);
100 }
101
102
103 /* UDataInfo cf. udata.h */
104 static UDataInfo dataInfo = {
105 sizeof(UDataInfo),
106 0,
107
108 U_IS_BIG_ENDIAN,
109 U_CHARSET_FAMILY,
110 U_SIZEOF_UCHAR,
111 0,
112
113 { 0x44, 0x69, 0x63, 0x74 }, /* "Dict" */
114 { 1, 0, 0, 0 }, /* format version */
115 { 0, 0, 0, 0 } /* data version */
116 };
117
118 #if !UCONFIG_NO_BREAK_ITERATION
119
120 // A wrapper for both BytesTrieBuilder and UCharsTrieBuilder.
121 // may want to put this somewhere in ICU, as it could be useful outside
122 // of this tool?
123 class DataDict {
124 private:
125 BytesTrieBuilder *bt;
126 UCharsTrieBuilder *ut;
127 UChar32 transformConstant;
128 int32_t transformType;
129 public:
130 // constructs a new data dictionary. if there is an error,
131 // it will be returned in status
132 // isBytesTrie != 0 will produce a BytesTrieBuilder,
133 // isBytesTrie == 0 will produce a UCharsTrieBuilder
DataDict(UBool isBytesTrie,UErrorCode & status)134 DataDict(UBool isBytesTrie, UErrorCode &status) : bt(nullptr), ut(nullptr),
135 transformConstant(0), transformType(DictionaryData::TRANSFORM_NONE) {
136 if (isBytesTrie) {
137 bt = new BytesTrieBuilder(status);
138 } else {
139 ut = new UCharsTrieBuilder(status);
140 }
141 }
142
~DataDict()143 ~DataDict() {
144 delete bt;
145 delete ut;
146 }
147
148 private:
transform(UChar32 c,UErrorCode & status)149 char transform(UChar32 c, UErrorCode &status) {
150 if (transformType == DictionaryData::TRANSFORM_TYPE_OFFSET) {
151 if (c == 0x200D) { return static_cast<char>(0xFF); }
152 else if (c == 0x200C) { return static_cast<char>(0xFE); }
153 int32_t delta = c - transformConstant;
154 if (delta < 0 || 0xFD < delta) {
155 fprintf(stderr, "Codepoint U+%04lx out of range for --transform offset-%04lx!\n",
156 static_cast<long>(c), static_cast<long>(transformConstant));
157 exit(U_ILLEGAL_ARGUMENT_ERROR); // TODO: should return and print the line number
158 }
159 return static_cast<char>(delta);
160 } else { // no such transform type
161 status = U_INTERNAL_PROGRAM_ERROR;
162 return static_cast<char>(c); // it should be noted this transform type will not generally work
163 }
164 }
165
transform(const UnicodeString & word,CharString & buf,UErrorCode & errorCode)166 void transform(const UnicodeString &word, CharString &buf, UErrorCode &errorCode) {
167 UChar32 c = 0;
168 int32_t len = word.length();
169 for (int32_t i = 0; i < len; i += U16_LENGTH(c)) {
170 c = word.char32At(i);
171 buf.append(transform(c, errorCode), errorCode);
172 }
173 }
174
175 public:
176 // sets the desired transformation data.
177 // should be populated from a command line argument
178 // so far the only acceptable format is offset-<hex constant>
179 // eventually others (mask-<hex constant>?) may be enabled
180 // more complex functions may be more difficult
setTransform(const char * t)181 void setTransform(const char *t) {
182 if (strncmp(t, "offset-", 7) == 0) {
183 char *end;
184 unsigned long base = uprv_strtoul(t + 7, &end, 16);
185 if (end == (t + 7) || *end != 0 || base > 0x10FF80) {
186 fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7);
187 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
188 }
189 transformType = DictionaryData::TRANSFORM_TYPE_OFFSET;
190 transformConstant = static_cast<UChar32>(base);
191 }
192 else {
193 fprintf(stderr, "Invalid transform specified: %s\n", t);
194 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
195 }
196 }
197
198 // add a word to the trie
addWord(const UnicodeString & word,int32_t value,UErrorCode & status)199 void addWord(const UnicodeString &word, int32_t value, UErrorCode &status) {
200 if (bt) {
201 CharString buf;
202 transform(word, buf, status);
203 bt->add(buf.toStringPiece(), value, status);
204 }
205 if (ut) { ut->add(word, value, status); }
206 }
207
208 // if we are a bytestrie, give back the StringPiece representing the serialized version of us
serializeBytes(UErrorCode & status)209 StringPiece serializeBytes(UErrorCode &status) {
210 return bt->buildStringPiece(USTRINGTRIE_BUILD_SMALL, status);
211 }
212
213 // if we are a ucharstrie, produce the UnicodeString representing the serialized version of us
serializeUChars(UnicodeString & s,UErrorCode & status)214 void serializeUChars(UnicodeString &s, UErrorCode &status) {
215 ut->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, s, status);
216 }
217
getTransform()218 int32_t getTransform() {
219 return (transformType | transformConstant);
220 }
221 };
222 #endif
223
224 static const char16_t LINEFEED_CHARACTER = 0x000A;
225 static const char16_t CARRIAGE_RETURN_CHARACTER = 0x000D;
226
readLine(UCHARBUF * f,UnicodeString & fileLine,IcuToolErrorCode & errorCode)227 static UBool readLine(UCHARBUF *f, UnicodeString &fileLine, IcuToolErrorCode &errorCode) {
228 int32_t lineLength;
229 const char16_t *line = ucbuf_readline(f, &lineLength, errorCode);
230 if(line == nullptr || errorCode.isFailure()) { return false; }
231 // Strip trailing CR/LF, comments, and spaces.
232 const char16_t *comment = u_memchr(line, 0x23, lineLength); // '#'
233 if(comment != nullptr) {
234 lineLength = static_cast<int32_t>(comment - line);
235 } else {
236 while(lineLength > 0 && (line[lineLength - 1] == CARRIAGE_RETURN_CHARACTER || line[lineLength - 1] == LINEFEED_CHARACTER)) { --lineLength; }
237 }
238 while(lineLength > 0 && u_isspace(line[lineLength - 1])) { --lineLength; }
239 fileLine.setTo(false, line, lineLength);
240 return true;
241 }
242
243 //----------------------------------------------------------------------------
244 //
245 // main for gendict
246 //
247 //----------------------------------------------------------------------------
main(int argc,char ** argv)248 int main(int argc, char **argv) {
249 //
250 // Pick up and check the command line arguments,
251 // using the standard ICU tool utils option handling.
252 //
253 U_MAIN_INIT_ARGS(argc, argv);
254 progName = argv[0];
255 argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
256 if(argc<0) {
257 // Unrecognized option
258 fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
259 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
260 }
261
262 if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) {
263 // -? or -h for help.
264 usageAndDie(U_ZERO_ERROR);
265 }
266
267 UBool verbose = options[ARG_VERBOSE].doesOccur;
268 UBool quiet = options[ARG_QUIET].doesOccur;
269
270 if (argc < 3) {
271 fprintf(stderr, "input and output file must both be specified.\n");
272 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
273 }
274 const char *outFileName = argv[2];
275 const char *wordFileName = argv[1];
276
277 startTime = uprv_getRawUTCtime(); // initialize start timer
278
279 if (options[ARG_ICUDATADIR].doesOccur) {
280 u_setDataDirectory(options[ARG_ICUDATADIR].value);
281 }
282
283 const char *copyright = nullptr;
284 if (options[ARG_COPYRIGHT].doesOccur) {
285 copyright = U_COPYRIGHT_STRING;
286 }
287
288 if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) {
289 fprintf(stderr, "you must specify exactly one type of trie to output!\n");
290 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
291 }
292 UBool isBytesTrie = options[ARG_BYTES].doesOccur;
293 if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) {
294 fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n");
295 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
296 }
297
298 IcuToolErrorCode status("gendict/main()");
299
300 UBool isToml = options[ARG_TOML].doesOccur;
301
302 #if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
303 const char* outDir=nullptr;
304
305 UNewDataMemory *pData;
306 char msg[1024];
307 UErrorCode tempstatus = U_ZERO_ERROR;
308
309 /* write message with just the name */ // potential for a buffer overflow here...
310 snprintf(msg, sizeof(msg), "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
311 fprintf(stderr, "%s\n", msg);
312
313 /* write the dummy data file */
314 pData = udata_create(outDir, nullptr, outFileName, &dataInfo, nullptr, &tempstatus);
315 udata_writeBlock(pData, msg, strlen(msg));
316 udata_finish(pData, &tempstatus);
317 return (int)tempstatus;
318
319 #else
320 // Read in the dictionary source file
321 if (verbose) { printf("Opening file %s...\n", wordFileName); }
322 const char *codepage = "UTF-8";
323 LocalUCHARBUFPointer f(ucbuf_open(wordFileName, &codepage, true, false, status));
324 if (status.isFailure()) {
325 fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName());
326 exit(status.reset());
327 }
328 if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); }
329 DataDict dict(isBytesTrie, status);
330 if (status.isFailure()) {
331 fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName());
332 exit(status.reset());
333 }
334 if (options[ARG_TRANSFORM].doesOccur) {
335 dict.setTransform(options[ARG_TRANSFORM].value);
336 }
337
338 UnicodeString fileLine;
339 if (verbose) { puts("Adding words to dictionary..."); }
340 UBool hasValues = false;
341 UBool hasValuelessContents = false;
342 int lineCount = 0;
343 int wordCount = 0;
344 int minlen = 255;
345 int maxlen = 0;
346 UBool isOk = true;
347 while (readLine(f.getAlias(), fileLine, status)) {
348 lineCount++;
349 if (fileLine.isEmpty()) continue;
350
351 // Parse word [spaces value].
352 int32_t keyLen;
353 for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {}
354 if (keyLen == 0) {
355 fprintf(stderr, "Error: no word on line %i!\n", lineCount);
356 isOk = false;
357 continue;
358 }
359 int32_t valueStart;
360 for (valueStart = keyLen;
361 valueStart < fileLine.length() && u_isspace(fileLine[valueStart]);
362 ++valueStart) {}
363
364 if (keyLen < valueStart) {
365 int32_t valueLength = fileLine.length() - valueStart;
366 if (valueLength > 15) {
367 fprintf(stderr, "Error: value too long on line %i!\n", lineCount);
368 isOk = false;
369 continue;
370 }
371 char s[16];
372 fileLine.extract(valueStart, valueLength, s, 16, US_INV);
373 char *end;
374 unsigned long value = uprv_strtoul(s, &end, 0);
375 if (end == s || *end != 0 || static_cast<int32_t>(uprv_strlen(s)) != valueLength || value > 0xffffffff) {
376 fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount);
377 isOk = false;
378 continue;
379 }
380 dict.addWord(fileLine.tempSubString(0, keyLen), static_cast<int32_t>(value), status);
381 hasValues = true;
382 wordCount++;
383 if (keyLen < minlen) minlen = keyLen;
384 if (keyLen > maxlen) maxlen = keyLen;
385 } else {
386 dict.addWord(fileLine.tempSubString(0, keyLen), 0, status);
387 hasValuelessContents = true;
388 wordCount++;
389 if (keyLen < minlen) minlen = keyLen;
390 if (keyLen > maxlen) maxlen = keyLen;
391 }
392
393 if (status.isFailure()) {
394 fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n",
395 status.errorName(), lineCount);
396 exit(status.reset());
397 }
398 }
399 if (verbose) { printf("Processed %d lines, added %d words, minlen %d, maxlen %d\n", lineCount, wordCount, minlen, maxlen); }
400
401 if (!isOk && status.isSuccess()) {
402 status.set(U_ILLEGAL_ARGUMENT_ERROR);
403 }
404 if (hasValues && hasValuelessContents) {
405 fprintf(stderr, "warning: file contained both valued and unvalued strings!\n");
406 }
407
408 if (verbose) { printf("Serializing data...isBytesTrie? %d\n", isBytesTrie); }
409 int32_t outDataSize;
410 const void *outData;
411 UnicodeString usp;
412 if (isBytesTrie) {
413 StringPiece sp = dict.serializeBytes(status);
414 outDataSize = sp.size();
415 outData = sp.data();
416 } else {
417 dict.serializeUChars(usp, status);
418 outDataSize = usp.length() * U_SIZEOF_UCHAR;
419 outData = usp.getBuffer();
420 }
421 if (status.isFailure()) {
422 fprintf(stderr, "gendict: got failure of type %s while serializing, if U_ILLEGAL_ARGUMENT_ERROR possibly due to duplicate dictionary entries\n", status.errorName());
423 exit(status.reset());
424 }
425 if (verbose) { puts("Opening output file..."); }
426
427 if (isToml) {
428 FILE* f = fopen(outFileName, "w");
429 if (f == nullptr) {
430 fprintf(stderr, "gendict: could not open output file \"%s\"\n", outFileName);
431 exit(status.reset());
432 }
433 fprintf(f, "trie_type = \"%s\"\n", isBytesTrie ? "bytes" : "uchars");
434 fprintf(f, "has_values = %s\n", hasValues ? "true" : "false");
435 int32_t transform = dict.getTransform();
436 bool isOffset = (transform & DictionaryData::TRANSFORM_TYPE_MASK) == DictionaryData::TRANSFORM_TYPE_OFFSET;
437 int32_t offset = transform & DictionaryData::TRANSFORM_OFFSET_MASK;
438 fprintf(f, "transform_type = \"%s\"\n", isOffset ? "offset" : "none");
439 fprintf(f, "transform_offset = %d\n", offset);
440
441 int32_t outDataWidth = isBytesTrie ? 8 : 16;
442 int32_t outDataLength = isBytesTrie ? outDataSize : outDataSize / U_SIZEOF_UCHAR;
443 usrc_writeArray(f, "trie_data = [\n ", outData, outDataWidth, outDataLength, " ", "\n]\n");
444
445
446 fclose(f);
447 } else {
448 UNewDataMemory *pData = udata_create(nullptr, nullptr, outFileName, &dataInfo, copyright, status);
449 if (status.isFailure()) {
450 fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName());
451 exit(status.reset());
452 }
453
454 if (verbose) { puts("Writing to output file..."); }
455 int32_t indexes[DictionaryData::IX_COUNT] = {
456 DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0
457 };
458 int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
459 indexes[DictionaryData::IX_RESERVED1_OFFSET] = size;
460 indexes[DictionaryData::IX_RESERVED2_OFFSET] = size;
461 indexes[DictionaryData::IX_TOTAL_SIZE] = size;
462
463 indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS;
464 if (hasValues) {
465 indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES;
466 }
467
468 indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform();
469 udata_writeBlock(pData, indexes, sizeof(indexes));
470 udata_writeBlock(pData, outData, outDataSize);
471 size_t bytesWritten = udata_finish(pData, status);
472 if (status.isFailure()) {
473 fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName());
474 exit(status.reset());
475 }
476
477 if (bytesWritten != static_cast<size_t>(size)) {
478 fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
479 exit(U_INTERNAL_PROGRAM_ERROR);
480 }
481 }
482
483 if (!quiet) { printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime()); }
484
485 #ifdef TEST_GENDICT
486 if (isBytesTrie) {
487 BytesTrie::Iterator it(outData, outDataSize, status);
488 while (it.hasNext()) {
489 it.next(status);
490 const StringPiece s = it.getString();
491 int32_t val = it.getValue();
492 printf("%s -> %i\n", s.data(), val);
493 }
494 } else {
495 UCharsTrie::Iterator it((const char16_t *)outData, outDataSize, status);
496 while (it.hasNext()) {
497 it.next(status);
498 const UnicodeString s = it.getString();
499 int32_t val = it.getValue();
500 char tmp[1024];
501 s.extract(0, s.length(), tmp, 1024);
502 printf("%s -> %i\n", tmp, val);
503 }
504 }
505 #endif
506
507 return 0;
508 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
509 }
510