1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 **********************************************************************
5 * Copyright (C) 2002-2016, International Business Machines
6 * Corporation and others. All Rights Reserved.
7 **********************************************************************
8 *
9 * File gendict.cpp
10 */
11
12 #include "unicode/utypes.h"
13 #include "unicode/uchar.h"
14 #include "unicode/ucnv.h"
15 #include "unicode/uniset.h"
16 #include "unicode/unistr.h"
17 #include "unicode/uclean.h"
18 #include "unicode/udata.h"
19 #include "unicode/putil.h"
20 #include "unicode/ucharstriebuilder.h"
21 #include "unicode/bytestriebuilder.h"
22 #include "unicode/ucharstrie.h"
23 #include "unicode/bytestrie.h"
24 #include "unicode/ucnv.h"
25 #include "unicode/utf16.h"
26
27 #include "charstr.h"
28 #include "dictionarydata.h"
29 #include "uoptions.h"
30 #include "unewdata.h"
31 #include "cmemory.h"
32 #include "uassert.h"
33 #include "ucbuf.h"
34 #include "toolutil.h"
35 #include "cstring.h"
36
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <string.h>
40
41 #include "putilimp.h"
42 UDate startTime;
43
elapsedTime()44 static int elapsedTime() {
45 return (int)uprv_floor((uprv_getRawUTCtime()-startTime)/1000.0);
46 }
47
48 U_NAMESPACE_USE
49
50 static char *progName;
51 static UOption options[]={
52 UOPTION_HELP_H, /* 0 */
53 UOPTION_HELP_QUESTION_MARK, /* 1 */
54 UOPTION_VERBOSE, /* 2 */
55 UOPTION_ICUDATADIR, /* 4 */
56 UOPTION_COPYRIGHT, /* 5 */
57 { "uchars", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 6 */
58 { "bytes", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 7 */
59 { "transform", NULL, NULL, NULL, '\1', UOPT_REQUIRES_ARG, 0}, /* 8 */
60 UOPTION_QUIET, /* 9 */
61 };
62
63 enum arguments {
64 ARG_HELP = 0,
65 ARG_QMARK,
66 ARG_VERBOSE,
67 ARG_ICUDATADIR,
68 ARG_COPYRIGHT,
69 ARG_UCHARS,
70 ARG_BYTES,
71 ARG_TRANSFORM,
72 ARG_QUIET
73 };
74
75 // prints out the standard usage method describing command line arguments,
76 // then bails out with the desired exit code
usageAndDie(UErrorCode retCode)77 static void usageAndDie(UErrorCode retCode) {
78 fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName);
79 fprintf((U_SUCCESS(retCode) ? stdout : stderr),
80 "\tRead in a word list and write out a string trie dictionary\n"
81 "options:\n"
82 "\t-h or -? or --help this usage text\n"
83 "\t-V or --version show a version message\n"
84 "\t-c or --copyright include a copyright notice\n"
85 "\t-v or --verbose turn on verbose output\n"
86 "\t-q or --quiet do not display warnings and progress\n"
87 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option
88 "\t followed by path, defaults to %s\n"
89 "\t--uchars output a UCharsTrie (mutually exclusive with -b!)\n"
90 "\t--bytes output a BytesTrie (mutually exclusive with -u!)\n"
91 "\t--transform the kind of transform to use (eg --transform offset-40A3,\n"
92 "\t which specifies an offset transform with constant 0x40A3)\n",
93 u_getDataDirectory());
94 exit(retCode);
95 }
96
97
98 /* UDataInfo cf. udata.h */
99 static UDataInfo dataInfo = {
100 sizeof(UDataInfo),
101 0,
102
103 U_IS_BIG_ENDIAN,
104 U_CHARSET_FAMILY,
105 U_SIZEOF_UCHAR,
106 0,
107
108 { 0x44, 0x69, 0x63, 0x74 }, /* "Dict" */
109 { 1, 0, 0, 0 }, /* format version */
110 { 0, 0, 0, 0 } /* data version */
111 };
112
113 #if !UCONFIG_NO_BREAK_ITERATION
114
115 // A wrapper for both BytesTrieBuilder and UCharsTrieBuilder.
116 // may want to put this somewhere in ICU, as it could be useful outside
117 // of this tool?
118 class DataDict {
119 private:
120 BytesTrieBuilder *bt;
121 UCharsTrieBuilder *ut;
122 UChar32 transformConstant;
123 int32_t transformType;
124 public:
125 // constructs a new data dictionary. if there is an error,
126 // it will be returned in status
127 // isBytesTrie != 0 will produce a BytesTrieBuilder,
128 // isBytesTrie == 0 will produce a UCharsTrieBuilder
DataDict(UBool isBytesTrie,UErrorCode & status)129 DataDict(UBool isBytesTrie, UErrorCode &status) : bt(NULL), ut(NULL),
130 transformConstant(0), transformType(DictionaryData::TRANSFORM_NONE) {
131 if (isBytesTrie) {
132 bt = new BytesTrieBuilder(status);
133 } else {
134 ut = new UCharsTrieBuilder(status);
135 }
136 }
137
~DataDict()138 ~DataDict() {
139 delete bt;
140 delete ut;
141 }
142
143 private:
transform(UChar32 c,UErrorCode & status)144 char transform(UChar32 c, UErrorCode &status) {
145 if (transformType == DictionaryData::TRANSFORM_TYPE_OFFSET) {
146 if (c == 0x200D) { return (char)0xFF; }
147 else if (c == 0x200C) { return (char)0xFE; }
148 int32_t delta = c - transformConstant;
149 if (delta < 0 || 0xFD < delta) {
150 fprintf(stderr, "Codepoint U+%04lx out of range for --transform offset-%04lx!\n",
151 (long)c, (long)transformConstant);
152 exit(U_ILLEGAL_ARGUMENT_ERROR); // TODO: should return and print the line number
153 }
154 return (char)delta;
155 } else { // no such transform type
156 status = U_INTERNAL_PROGRAM_ERROR;
157 return (char)c; // it should be noted this transform type will not generally work
158 }
159 }
160
transform(const UnicodeString & word,CharString & buf,UErrorCode & errorCode)161 void transform(const UnicodeString &word, CharString &buf, UErrorCode &errorCode) {
162 UChar32 c = 0;
163 int32_t len = word.length();
164 for (int32_t i = 0; i < len; i += U16_LENGTH(c)) {
165 c = word.char32At(i);
166 buf.append(transform(c, errorCode), errorCode);
167 }
168 }
169
170 public:
171 // sets the desired transformation data.
172 // should be populated from a command line argument
173 // so far the only acceptable format is offset-<hex constant>
174 // eventually others (mask-<hex constant>?) may be enabled
175 // more complex functions may be more difficult
setTransform(const char * t)176 void setTransform(const char *t) {
177 if (strncmp(t, "offset-", 7) == 0) {
178 char *end;
179 unsigned long base = uprv_strtoul(t + 7, &end, 16);
180 if (end == (t + 7) || *end != 0 || base > 0x10FF80) {
181 fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7);
182 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
183 }
184 transformType = DictionaryData::TRANSFORM_TYPE_OFFSET;
185 transformConstant = (UChar32)base;
186 }
187 else {
188 fprintf(stderr, "Invalid transform specified: %s\n", t);
189 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
190 }
191 }
192
193 // add a word to the trie
addWord(const UnicodeString & word,int32_t value,UErrorCode & status)194 void addWord(const UnicodeString &word, int32_t value, UErrorCode &status) {
195 if (bt) {
196 CharString buf;
197 transform(word, buf, status);
198 bt->add(buf.toStringPiece(), value, status);
199 }
200 if (ut) { ut->add(word, value, status); }
201 }
202
203 // if we are a bytestrie, give back the StringPiece representing the serialized version of us
serializeBytes(UErrorCode & status)204 StringPiece serializeBytes(UErrorCode &status) {
205 return bt->buildStringPiece(USTRINGTRIE_BUILD_SMALL, status);
206 }
207
208 // if we are a ucharstrie, produce the UnicodeString representing the serialized version of us
serializeUChars(UnicodeString & s,UErrorCode & status)209 void serializeUChars(UnicodeString &s, UErrorCode &status) {
210 ut->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, s, status);
211 }
212
getTransform()213 int32_t getTransform() {
214 return (int32_t)(transformType | transformConstant);
215 }
216 };
217 #endif
218
219 static const UChar LINEFEED_CHARACTER = 0x000A;
220 static const UChar CARRIAGE_RETURN_CHARACTER = 0x000D;
221
readLine(UCHARBUF * f,UnicodeString & fileLine,IcuToolErrorCode & errorCode)222 static UBool readLine(UCHARBUF *f, UnicodeString &fileLine, IcuToolErrorCode &errorCode) {
223 int32_t lineLength;
224 const UChar *line = ucbuf_readline(f, &lineLength, errorCode);
225 if(line == NULL || errorCode.isFailure()) { return FALSE; }
226 // Strip trailing CR/LF, comments, and spaces.
227 const UChar *comment = u_memchr(line, 0x23, lineLength); // '#'
228 if(comment != NULL) {
229 lineLength = (int32_t)(comment - line);
230 } else {
231 while(lineLength > 0 && (line[lineLength - 1] == CARRIAGE_RETURN_CHARACTER || line[lineLength - 1] == LINEFEED_CHARACTER)) { --lineLength; }
232 }
233 while(lineLength > 0 && u_isspace(line[lineLength - 1])) { --lineLength; }
234 fileLine.setTo(FALSE, line, lineLength);
235 return TRUE;
236 }
237
238 //----------------------------------------------------------------------------
239 //
240 // main for gendict
241 //
242 //----------------------------------------------------------------------------
main(int argc,char ** argv)243 int main(int argc, char **argv) {
244 //
245 // Pick up and check the command line arguments,
246 // using the standard ICU tool utils option handling.
247 //
248 U_MAIN_INIT_ARGS(argc, argv);
249 progName = argv[0];
250 argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
251 if(argc<0) {
252 // Unrecognized option
253 fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
254 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
255 }
256
257 if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) {
258 // -? or -h for help.
259 usageAndDie(U_ZERO_ERROR);
260 }
261
262 UBool verbose = options[ARG_VERBOSE].doesOccur;
263 UBool quiet = options[ARG_QUIET].doesOccur;
264
265 if (argc < 3) {
266 fprintf(stderr, "input and output file must both be specified.\n");
267 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
268 }
269 const char *outFileName = argv[2];
270 const char *wordFileName = argv[1];
271
272 startTime = uprv_getRawUTCtime(); // initialize start timer
273
274 if (options[ARG_ICUDATADIR].doesOccur) {
275 u_setDataDirectory(options[ARG_ICUDATADIR].value);
276 }
277
278 const char *copyright = NULL;
279 if (options[ARG_COPYRIGHT].doesOccur) {
280 copyright = U_COPYRIGHT_STRING;
281 }
282
283 if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) {
284 fprintf(stderr, "you must specify exactly one type of trie to output!\n");
285 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
286 }
287 UBool isBytesTrie = options[ARG_BYTES].doesOccur;
288 if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) {
289 fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n");
290 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
291 }
292
293 IcuToolErrorCode status("gendict/main()");
294
295 #if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
296 const char* outDir=NULL;
297
298 UNewDataMemory *pData;
299 char msg[1024];
300 UErrorCode tempstatus = U_ZERO_ERROR;
301
302 /* write message with just the name */ // potential for a buffer overflow here...
303 sprintf(msg, "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
304 fprintf(stderr, "%s\n", msg);
305
306 /* write the dummy data file */
307 pData = udata_create(outDir, NULL, outFileName, &dataInfo, NULL, &tempstatus);
308 udata_writeBlock(pData, msg, strlen(msg));
309 udata_finish(pData, &tempstatus);
310 return (int)tempstatus;
311
312 #else
313 // Read in the dictionary source file
314 if (verbose) { printf("Opening file %s...\n", wordFileName); }
315 const char *codepage = "UTF-8";
316 UCHARBUF *f = ucbuf_open(wordFileName, &codepage, TRUE, FALSE, status);
317 if (status.isFailure()) {
318 fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName());
319 exit(status.reset());
320 }
321 if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); }
322 DataDict dict(isBytesTrie, status);
323 if (status.isFailure()) {
324 fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName());
325 exit(status.reset());
326 }
327 if (options[ARG_TRANSFORM].doesOccur) {
328 dict.setTransform(options[ARG_TRANSFORM].value);
329 }
330
331 UnicodeString fileLine;
332 if (verbose) { puts("Adding words to dictionary..."); }
333 UBool hasValues = FALSE;
334 UBool hasValuelessContents = FALSE;
335 int lineCount = 0;
336 int wordCount = 0;
337 int minlen = 255;
338 int maxlen = 0;
339 UBool isOk = TRUE;
340 while (readLine(f, fileLine, status)) {
341 lineCount++;
342 if (fileLine.isEmpty()) continue;
343
344 // Parse word [spaces value].
345 int32_t keyLen;
346 for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {}
347 if (keyLen == 0) {
348 fprintf(stderr, "Error: no word on line %i!\n", lineCount);
349 isOk = FALSE;
350 continue;
351 }
352 int32_t valueStart;
353 for (valueStart = keyLen;
354 valueStart < fileLine.length() && u_isspace(fileLine[valueStart]);
355 ++valueStart) {}
356
357 if (keyLen < valueStart) {
358 int32_t valueLength = fileLine.length() - valueStart;
359 if (valueLength > 15) {
360 fprintf(stderr, "Error: value too long on line %i!\n", lineCount);
361 isOk = FALSE;
362 continue;
363 }
364 char s[16];
365 fileLine.extract(valueStart, valueLength, s, 16, US_INV);
366 char *end;
367 unsigned long value = uprv_strtoul(s, &end, 0);
368 if (end == s || *end != 0 || (int32_t)uprv_strlen(s) != valueLength || value > 0xffffffff) {
369 fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount);
370 isOk = FALSE;
371 continue;
372 }
373 dict.addWord(fileLine.tempSubString(0, keyLen), (int32_t)value, status);
374 hasValues = TRUE;
375 wordCount++;
376 if (keyLen < minlen) minlen = keyLen;
377 if (keyLen > maxlen) maxlen = keyLen;
378 } else {
379 dict.addWord(fileLine.tempSubString(0, keyLen), 0, status);
380 hasValuelessContents = TRUE;
381 wordCount++;
382 if (keyLen < minlen) minlen = keyLen;
383 if (keyLen > maxlen) maxlen = keyLen;
384 }
385
386 if (status.isFailure()) {
387 fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n",
388 status.errorName(), lineCount);
389 exit(status.reset());
390 }
391 }
392 if (verbose) { printf("Processed %d lines, added %d words, minlen %d, maxlen %d\n", lineCount, wordCount, minlen, maxlen); }
393
394 if (!isOk && status.isSuccess()) {
395 status.set(U_ILLEGAL_ARGUMENT_ERROR);
396 }
397 if (hasValues && hasValuelessContents) {
398 fprintf(stderr, "warning: file contained both valued and unvalued strings!\n");
399 }
400
401 if (verbose) { printf("Serializing data...isBytesTrie? %d\n", isBytesTrie); }
402 int32_t outDataSize;
403 const void *outData;
404 UnicodeString usp;
405 if (isBytesTrie) {
406 StringPiece sp = dict.serializeBytes(status);
407 outDataSize = sp.size();
408 outData = sp.data();
409 } else {
410 dict.serializeUChars(usp, status);
411 outDataSize = usp.length() * U_SIZEOF_UCHAR;
412 outData = usp.getBuffer();
413 }
414 if (status.isFailure()) {
415 fprintf(stderr, "gendict: got failure of type %s while serializing, if U_ILLEGAL_ARGUMENT_ERROR possibly due to duplicate dictionary entries\n", status.errorName());
416 exit(status.reset());
417 }
418 if (verbose) { puts("Opening output file..."); }
419 UNewDataMemory *pData = udata_create(NULL, NULL, outFileName, &dataInfo, copyright, status);
420 if (status.isFailure()) {
421 fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName());
422 exit(status.reset());
423 }
424
425 if (verbose) { puts("Writing to output file..."); }
426 int32_t indexes[DictionaryData::IX_COUNT] = {
427 DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0
428 };
429 int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
430 indexes[DictionaryData::IX_RESERVED1_OFFSET] = size;
431 indexes[DictionaryData::IX_RESERVED2_OFFSET] = size;
432 indexes[DictionaryData::IX_TOTAL_SIZE] = size;
433
434 indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS;
435 if (hasValues) {
436 indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES;
437 }
438
439 indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform();
440 udata_writeBlock(pData, indexes, sizeof(indexes));
441 udata_writeBlock(pData, outData, outDataSize);
442 size_t bytesWritten = udata_finish(pData, status);
443 if (status.isFailure()) {
444 fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName());
445 exit(status.reset());
446 }
447
448 if (bytesWritten != (size_t)size) {
449 fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
450 exit(U_INTERNAL_PROGRAM_ERROR);
451 }
452
453 if (!quiet) { printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime()); }
454
455 #ifdef TEST_GENDICT
456 if (isBytesTrie) {
457 BytesTrie::Iterator it(outData, outDataSize, status);
458 while (it.hasNext()) {
459 it.next(status);
460 const StringPiece s = it.getString();
461 int32_t val = it.getValue();
462 printf("%s -> %i\n", s.data(), val);
463 }
464 } else {
465 UCharsTrie::Iterator it((const UChar *)outData, outDataSize, status);
466 while (it.hasNext()) {
467 it.next(status);
468 const UnicodeString s = it.getString();
469 int32_t val = it.getValue();
470 char tmp[1024];
471 s.extract(0, s.length(), tmp, 1024);
472 printf("%s -> %i\n", tmp, val);
473 }
474 }
475 #endif
476
477 return 0;
478 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
479 }
480