• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 **********************************************************************
5 *   Copyright (C) 2002-2016, International Business Machines
6 *   Corporation and others.  All Rights Reserved.
7 **********************************************************************
8 *
9 * File gendict.cpp
10 */
11 
12 #include "unicode/utypes.h"
13 #include "unicode/uchar.h"
14 #include "unicode/ucnv.h"
15 #include "unicode/uniset.h"
16 #include "unicode/unistr.h"
17 #include "unicode/uclean.h"
18 #include "unicode/udata.h"
19 #include "unicode/putil.h"
20 #include "unicode/ucharstriebuilder.h"
21 #include "unicode/bytestriebuilder.h"
22 #include "unicode/ucharstrie.h"
23 #include "unicode/bytestrie.h"
24 #include "unicode/ucnv.h"
25 #include "unicode/ustring.h"
26 #include "unicode/utf16.h"
27 
28 #include "charstr.h"
29 #include "dictionarydata.h"
30 #include "uoptions.h"
31 #include "unewdata.h"
32 #include "cmemory.h"
33 #include "uassert.h"
34 #include "ucbuf.h"
35 #include "toolutil.h"
36 #include "cstring.h"
37 
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 
42 #include "putilimp.h"
43 UDate startTime;
44 
elapsedTime()45 static int elapsedTime() {
46   return (int)uprv_floor((uprv_getRawUTCtime()-startTime)/1000.0);
47 }
48 
49 U_NAMESPACE_USE
50 
51 static char *progName;
52 static UOption options[]={
53     UOPTION_HELP_H,             /* 0 */
54     UOPTION_HELP_QUESTION_MARK, /* 1 */
55     UOPTION_VERBOSE,            /* 2 */
56     UOPTION_ICUDATADIR,         /* 4 */
57     UOPTION_COPYRIGHT,          /* 5 */
58     { "uchars", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 6 */
59     { "bytes", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 7 */
60     { "transform", NULL, NULL, NULL, '\1', UOPT_REQUIRES_ARG, 0}, /* 8 */
61     UOPTION_QUIET,              /* 9 */
62 };
63 
64 enum arguments {
65     ARG_HELP = 0,
66     ARG_QMARK,
67     ARG_VERBOSE,
68     ARG_ICUDATADIR,
69     ARG_COPYRIGHT,
70     ARG_UCHARS,
71     ARG_BYTES,
72     ARG_TRANSFORM,
73     ARG_QUIET
74 };
75 
76 // prints out the standard usage method describing command line arguments,
77 // then bails out with the desired exit code
usageAndDie(UErrorCode retCode)78 static void usageAndDie(UErrorCode retCode) {
79     fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName);
80     fprintf((U_SUCCESS(retCode) ? stdout : stderr),
81            "\tRead in a word list and write out a string trie dictionary\n"
82            "options:\n"
83            "\t-h or -? or --help  this usage text\n"
84            "\t-V or --version     show a version message\n"
85            "\t-c or --copyright   include a copyright notice\n"
86            "\t-v or --verbose     turn on verbose output\n"
87            "\t-q or --quiet       do not display warnings and progress\n"
88            "\t-i or --icudatadir  directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option
89            "\t                    followed by path, defaults to %s\n"
90            "\t--uchars            output a UCharsTrie (mutually exclusive with -b!)\n"
91            "\t--bytes             output a BytesTrie (mutually exclusive with -u!)\n"
92            "\t--transform         the kind of transform to use (eg --transform offset-40A3,\n"
93            "\t                    which specifies an offset transform with constant 0x40A3)\n",
94             u_getDataDirectory());
95     exit(retCode);
96 }
97 
98 
99 /* UDataInfo cf. udata.h */
100 static UDataInfo dataInfo = {
101     sizeof(UDataInfo),
102     0,
103 
104     U_IS_BIG_ENDIAN,
105     U_CHARSET_FAMILY,
106     U_SIZEOF_UCHAR,
107     0,
108 
109     { 0x44, 0x69, 0x63, 0x74 },     /* "Dict" */
110     { 1, 0, 0, 0 },                 /* format version */
111     { 0, 0, 0, 0 }                  /* data version */
112 };
113 
114 #if !UCONFIG_NO_BREAK_ITERATION
115 
116 // A wrapper for both BytesTrieBuilder and UCharsTrieBuilder.
117 // may want to put this somewhere in ICU, as it could be useful outside
118 // of this tool?
119 class DataDict {
120 private:
121     BytesTrieBuilder *bt;
122     UCharsTrieBuilder *ut;
123     UChar32 transformConstant;
124     int32_t transformType;
125 public:
126     // constructs a new data dictionary. if there is an error,
127     // it will be returned in status
128     // isBytesTrie != 0 will produce a BytesTrieBuilder,
129     // isBytesTrie == 0 will produce a UCharsTrieBuilder
DataDict(UBool isBytesTrie,UErrorCode & status)130     DataDict(UBool isBytesTrie, UErrorCode &status) : bt(NULL), ut(NULL),
131         transformConstant(0), transformType(DictionaryData::TRANSFORM_NONE) {
132         if (isBytesTrie) {
133             bt = new BytesTrieBuilder(status);
134         } else {
135             ut = new UCharsTrieBuilder(status);
136         }
137     }
138 
~DataDict()139     ~DataDict() {
140         delete bt;
141         delete ut;
142     }
143 
144 private:
transform(UChar32 c,UErrorCode & status)145     char transform(UChar32 c, UErrorCode &status) {
146         if (transformType == DictionaryData::TRANSFORM_TYPE_OFFSET) {
147             if (c == 0x200D) { return (char)0xFF; }
148             else if (c == 0x200C) { return (char)0xFE; }
149             int32_t delta = c - transformConstant;
150             if (delta < 0 || 0xFD < delta) {
151                 fprintf(stderr, "Codepoint U+%04lx out of range for --transform offset-%04lx!\n",
152                         (long)c, (long)transformConstant);
153                 exit(U_ILLEGAL_ARGUMENT_ERROR); // TODO: should return and print the line number
154             }
155             return (char)delta;
156         } else { // no such transform type
157             status = U_INTERNAL_PROGRAM_ERROR;
158             return (char)c; // it should be noted this transform type will not generally work
159         }
160     }
161 
transform(const UnicodeString & word,CharString & buf,UErrorCode & errorCode)162     void transform(const UnicodeString &word, CharString &buf, UErrorCode &errorCode) {
163         UChar32 c = 0;
164         int32_t len = word.length();
165         for (int32_t i = 0; i < len; i += U16_LENGTH(c)) {
166             c = word.char32At(i);
167             buf.append(transform(c, errorCode), errorCode);
168         }
169     }
170 
171 public:
172     // sets the desired transformation data.
173     // should be populated from a command line argument
174     // so far the only acceptable format is offset-<hex constant>
175     // eventually others (mask-<hex constant>?) may be enabled
176     // more complex functions may be more difficult
setTransform(const char * t)177     void setTransform(const char *t) {
178         if (strncmp(t, "offset-", 7) == 0) {
179             char *end;
180             unsigned long base = uprv_strtoul(t + 7, &end, 16);
181             if (end == (t + 7) || *end != 0 || base > 0x10FF80) {
182                 fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7);
183                 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
184             }
185             transformType = DictionaryData::TRANSFORM_TYPE_OFFSET;
186             transformConstant = (UChar32)base;
187         }
188         else {
189             fprintf(stderr, "Invalid transform specified: %s\n", t);
190             usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
191         }
192     }
193 
194     // add a word to the trie
addWord(const UnicodeString & word,int32_t value,UErrorCode & status)195     void addWord(const UnicodeString &word, int32_t value, UErrorCode &status) {
196         if (bt) {
197             CharString buf;
198             transform(word, buf, status);
199             bt->add(buf.toStringPiece(), value, status);
200         }
201         if (ut) { ut->add(word, value, status); }
202     }
203 
204     // if we are a bytestrie, give back the StringPiece representing the serialized version of us
serializeBytes(UErrorCode & status)205     StringPiece serializeBytes(UErrorCode &status) {
206         return bt->buildStringPiece(USTRINGTRIE_BUILD_SMALL, status);
207     }
208 
209     // if we are a ucharstrie, produce the UnicodeString representing the serialized version of us
serializeUChars(UnicodeString & s,UErrorCode & status)210     void serializeUChars(UnicodeString &s, UErrorCode &status) {
211         ut->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, s, status);
212     }
213 
getTransform()214     int32_t getTransform() {
215         return (int32_t)(transformType | transformConstant);
216     }
217 };
218 #endif
219 
220 static const UChar LINEFEED_CHARACTER = 0x000A;
221 static const UChar CARRIAGE_RETURN_CHARACTER = 0x000D;
222 
readLine(UCHARBUF * f,UnicodeString & fileLine,IcuToolErrorCode & errorCode)223 static UBool readLine(UCHARBUF *f, UnicodeString &fileLine, IcuToolErrorCode &errorCode) {
224     int32_t lineLength;
225     const UChar *line = ucbuf_readline(f, &lineLength, errorCode);
226     if(line == NULL || errorCode.isFailure()) { return FALSE; }
227     // Strip trailing CR/LF, comments, and spaces.
228     const UChar *comment = u_memchr(line, 0x23, lineLength);  // '#'
229     if(comment != NULL) {
230         lineLength = (int32_t)(comment - line);
231     } else {
232         while(lineLength > 0 && (line[lineLength - 1] == CARRIAGE_RETURN_CHARACTER || line[lineLength - 1] == LINEFEED_CHARACTER)) { --lineLength; }
233     }
234     while(lineLength > 0 && u_isspace(line[lineLength - 1])) { --lineLength; }
235     fileLine.setTo(FALSE, line, lineLength);
236     return TRUE;
237 }
238 
239 //----------------------------------------------------------------------------
240 //
241 //  main      for gendict
242 //
243 //----------------------------------------------------------------------------
main(int argc,char ** argv)244 int  main(int argc, char **argv) {
245     //
246     // Pick up and check the command line arguments,
247     //    using the standard ICU tool utils option handling.
248     //
249     U_MAIN_INIT_ARGS(argc, argv);
250     progName = argv[0];
251     argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
252     if(argc<0) {
253         // Unrecognized option
254         fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
255         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
256     }
257 
258     if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) {
259         //  -? or -h for help.
260         usageAndDie(U_ZERO_ERROR);
261     }
262 
263     UBool verbose = options[ARG_VERBOSE].doesOccur;
264     UBool quiet = options[ARG_QUIET].doesOccur;
265 
266     if (argc < 3) {
267         fprintf(stderr, "input and output file must both be specified.\n");
268         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
269     }
270     const char *outFileName  = argv[2];
271     const char *wordFileName = argv[1];
272 
273     startTime = uprv_getRawUTCtime(); // initialize start timer
274 
275 	if (options[ARG_ICUDATADIR].doesOccur) {
276         u_setDataDirectory(options[ARG_ICUDATADIR].value);
277     }
278 
279     const char *copyright = NULL;
280     if (options[ARG_COPYRIGHT].doesOccur) {
281         copyright = U_COPYRIGHT_STRING;
282     }
283 
284     if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) {
285         fprintf(stderr, "you must specify exactly one type of trie to output!\n");
286         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
287     }
288     UBool isBytesTrie = options[ARG_BYTES].doesOccur;
289     if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) {
290         fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n");
291         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
292     }
293 
294     IcuToolErrorCode status("gendict/main()");
295 
296 #if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
297     const char* outDir=NULL;
298 
299     UNewDataMemory *pData;
300     char msg[1024];
301     UErrorCode tempstatus = U_ZERO_ERROR;
302 
303     /* write message with just the name */ // potential for a buffer overflow here...
304     sprintf(msg, "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
305     fprintf(stderr, "%s\n", msg);
306 
307     /* write the dummy data file */
308     pData = udata_create(outDir, NULL, outFileName, &dataInfo, NULL, &tempstatus);
309     udata_writeBlock(pData, msg, strlen(msg));
310     udata_finish(pData, &tempstatus);
311     return (int)tempstatus;
312 
313 #else
314     //  Read in the dictionary source file
315     if (verbose) { printf("Opening file %s...\n", wordFileName); }
316     const char *codepage = "UTF-8";
317     LocalUCHARBUFPointer f(ucbuf_open(wordFileName, &codepage, TRUE, FALSE, status));
318     if (status.isFailure()) {
319         fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName());
320         exit(status.reset());
321     }
322     if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); }
323     DataDict dict(isBytesTrie, status);
324     if (status.isFailure()) {
325         fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName());
326         exit(status.reset());
327     }
328     if (options[ARG_TRANSFORM].doesOccur) {
329         dict.setTransform(options[ARG_TRANSFORM].value);
330     }
331 
332     UnicodeString fileLine;
333     if (verbose) { puts("Adding words to dictionary..."); }
334     UBool hasValues = FALSE;
335     UBool hasValuelessContents = FALSE;
336     int lineCount = 0;
337     int wordCount = 0;
338     int minlen = 255;
339     int maxlen = 0;
340     UBool isOk = TRUE;
341     while (readLine(f.getAlias(), fileLine, status)) {
342         lineCount++;
343         if (fileLine.isEmpty()) continue;
344 
345         // Parse word [spaces value].
346         int32_t keyLen;
347         for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {}
348         if (keyLen == 0) {
349             fprintf(stderr, "Error: no word on line %i!\n", lineCount);
350             isOk = FALSE;
351             continue;
352         }
353         int32_t valueStart;
354         for (valueStart = keyLen;
355             valueStart < fileLine.length() && u_isspace(fileLine[valueStart]);
356             ++valueStart) {}
357 
358         if (keyLen < valueStart) {
359             int32_t valueLength = fileLine.length() - valueStart;
360             if (valueLength > 15) {
361                 fprintf(stderr, "Error: value too long on line %i!\n", lineCount);
362                 isOk = FALSE;
363                 continue;
364             }
365             char s[16];
366             fileLine.extract(valueStart, valueLength, s, 16, US_INV);
367             char *end;
368             unsigned long value = uprv_strtoul(s, &end, 0);
369             if (end == s || *end != 0 || (int32_t)uprv_strlen(s) != valueLength || value > 0xffffffff) {
370                 fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount);
371                 isOk = FALSE;
372                 continue;
373             }
374             dict.addWord(fileLine.tempSubString(0, keyLen), (int32_t)value, status);
375             hasValues = TRUE;
376             wordCount++;
377             if (keyLen < minlen) minlen = keyLen;
378             if (keyLen > maxlen) maxlen = keyLen;
379         } else {
380             dict.addWord(fileLine.tempSubString(0, keyLen), 0, status);
381             hasValuelessContents = TRUE;
382             wordCount++;
383             if (keyLen < minlen) minlen = keyLen;
384             if (keyLen > maxlen) maxlen = keyLen;
385         }
386 
387         if (status.isFailure()) {
388             fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n",
389                 status.errorName(), lineCount);
390             exit(status.reset());
391         }
392     }
393     if (verbose) { printf("Processed %d lines, added %d words, minlen %d, maxlen %d\n", lineCount, wordCount, minlen, maxlen); }
394 
395     if (!isOk && status.isSuccess()) {
396         status.set(U_ILLEGAL_ARGUMENT_ERROR);
397     }
398     if (hasValues && hasValuelessContents) {
399         fprintf(stderr, "warning: file contained both valued and unvalued strings!\n");
400     }
401 
402     if (verbose) { printf("Serializing data...isBytesTrie? %d\n", isBytesTrie); }
403     int32_t outDataSize;
404     const void *outData;
405     UnicodeString usp;
406     if (isBytesTrie) {
407         StringPiece sp = dict.serializeBytes(status);
408         outDataSize = sp.size();
409         outData = sp.data();
410     } else {
411         dict.serializeUChars(usp, status);
412         outDataSize = usp.length() * U_SIZEOF_UCHAR;
413         outData = usp.getBuffer();
414     }
415     if (status.isFailure()) {
416         fprintf(stderr, "gendict: got failure of type %s while serializing, if U_ILLEGAL_ARGUMENT_ERROR possibly due to duplicate dictionary entries\n", status.errorName());
417         exit(status.reset());
418     }
419     if (verbose) { puts("Opening output file..."); }
420     UNewDataMemory *pData = udata_create(NULL, NULL, outFileName, &dataInfo, copyright, status);
421     if (status.isFailure()) {
422         fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName());
423         exit(status.reset());
424     }
425 
426     if (verbose) { puts("Writing to output file..."); }
427     int32_t indexes[DictionaryData::IX_COUNT] = {
428         DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0
429     };
430     int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
431     indexes[DictionaryData::IX_RESERVED1_OFFSET] = size;
432     indexes[DictionaryData::IX_RESERVED2_OFFSET] = size;
433     indexes[DictionaryData::IX_TOTAL_SIZE] = size;
434 
435     indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS;
436     if (hasValues) {
437         indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES;
438     }
439 
440     indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform();
441     udata_writeBlock(pData, indexes, sizeof(indexes));
442     udata_writeBlock(pData, outData, outDataSize);
443     size_t bytesWritten = udata_finish(pData, status);
444     if (status.isFailure()) {
445         fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName());
446         exit(status.reset());
447     }
448 
449     if (bytesWritten != (size_t)size) {
450         fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
451         exit(U_INTERNAL_PROGRAM_ERROR);
452     }
453 
454     if (!quiet) { printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime()); }
455 
456 #ifdef TEST_GENDICT
457     if (isBytesTrie) {
458         BytesTrie::Iterator it(outData, outDataSize, status);
459         while (it.hasNext()) {
460             it.next(status);
461             const StringPiece s = it.getString();
462             int32_t val = it.getValue();
463             printf("%s -> %i\n", s.data(), val);
464         }
465     } else {
466         UCharsTrie::Iterator it((const UChar *)outData, outDataSize, status);
467         while (it.hasNext()) {
468             it.next(status);
469             const UnicodeString s = it.getString();
470             int32_t val = it.getValue();
471             char tmp[1024];
472             s.extract(0, s.length(), tmp, 1024);
473             printf("%s -> %i\n", tmp, val);
474         }
475     }
476 #endif
477 
478     return 0;
479 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
480 }
481