1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 2003-2012, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: gensprep.c
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 2003-02-06
14 * created by: Ram Viswanadha
15 *
16 * This program reads the Profile.txt files,
17 * parses them, and extracts the data for StringPrep profile.
18 * It then preprocesses it and writes a binary file for efficient use
19 * in various StringPrep conversion processes.
20 */
21
22 #define USPREP_TYPE_NAMES_ARRAY 1
23
24 #include <stdio.h>
25 #include <stdlib.h>
26
27 #include "cmemory.h"
28 #include "cstring.h"
29 #include "unewdata.h"
30 #include "uoptions.h"
31 #include "uparse.h"
32 #include "sprpimpl.h"
33
34 #include "unicode/uclean.h"
35 #include "unicode/udata.h"
36 #include "unicode/utypes.h"
37 #include "unicode/putil.h"
38
39
40 U_CDECL_BEGIN
41 #include "gensprep.h"
42 U_CDECL_END
43
44 UBool beVerbose=FALSE, haveCopyright=TRUE;
45
46 #define NORM_CORRECTIONS_FILE_NAME "NormalizationCorrections.txt"
47
48 #define NORMALIZE_DIRECTIVE "normalize"
49 #define NORMALIZE_DIRECTIVE_LEN 9
50 #define CHECK_BIDI_DIRECTIVE "check-bidi"
51 #define CHECK_BIDI_DIRECTIVE_LEN 10
52
53 /* prototypes --------------------------------------------------------------- */
54
55 static void
56 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode);
57
58 static void
59 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode);
60
61
62 /* -------------------------------------------------------------------------- */
63
64 static UOption options[]={
65 UOPTION_HELP_H,
66 UOPTION_HELP_QUESTION_MARK,
67 UOPTION_VERBOSE,
68 UOPTION_COPYRIGHT,
69 UOPTION_DESTDIR,
70 UOPTION_SOURCEDIR,
71 UOPTION_ICUDATADIR,
72 UOPTION_BUNDLE_NAME,
73 { "normalization", NULL, NULL, NULL, 'n', UOPT_REQUIRES_ARG, 0 },
74 { "norm-correction", NULL, NULL, NULL, 'm', UOPT_REQUIRES_ARG, 0 },
75 { "check-bidi", NULL, NULL, NULL, 'k', UOPT_NO_ARG, 0},
76 { "unicode", NULL, NULL, NULL, 'u', UOPT_REQUIRES_ARG, 0 },
77 };
78
79 enum{
80 HELP,
81 HELP_QUESTION_MARK,
82 VERBOSE,
83 COPYRIGHT,
84 DESTDIR,
85 SOURCEDIR,
86 ICUDATADIR,
87 BUNDLE_NAME,
88 NORMALIZE,
89 NORM_CORRECTION_DIR,
90 CHECK_BIDI,
91 UNICODE_VERSION
92 };
93
printHelp(int argc,char * argv[])94 static int printHelp(int argc, char* argv[]){
95 /*
96 * Broken into chucks because the C89 standard says the minimum
97 * required supported string length is 509 bytes.
98 */
99 fprintf(stderr,
100 "Usage: %s [-options] [file_name]\n"
101 "\n"
102 "Read the files specified and\n"
103 "create a binary file [package-name]_[bundle-name]." DATA_TYPE " with the StringPrep profile data\n"
104 "\n",
105 argv[0]);
106 fprintf(stderr,
107 "Options:\n"
108 "\t-h or -? or --help print this usage text\n"
109 "\t-v or --verbose verbose output\n"
110 "\t-c or --copyright include a copyright notice\n");
111 fprintf(stderr,
112 "\t-d or --destdir destination directory, followed by the path\n"
113 "\t-s or --sourcedir source directory of ICU data, followed by the path\n"
114 "\t-b or --bundle-name generate the ouput data file with the name specified\n"
115 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
116 "\t followed by path, defaults to %s\n",
117 u_getDataDirectory());
118 fprintf(stderr,
119 "\t-n or --normalize turn on the option for normalization and include mappings\n"
120 "\t from NormalizationCorrections.txt from the given path,\n"
121 "\t e.g: /test/icu/source/data/unidata\n");
122 fprintf(stderr,
123 "\t-m or --norm-correction use NormalizationCorrections.txt from the given path\n"
124 "\t when the input file contains a normalization directive.\n"
125 "\t unlike -n/--normalize, this option does not force the\n"
126 "\t normalization.\n");
127 fprintf(stderr,
128 "\t-k or --check-bidi turn on the option for checking for BiDi in the profile\n"
129 "\t-u or --unicode version of Unicode to be used with this profile followed by the version\n"
130 );
131 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
132 }
133
134
135 extern int
main(int argc,char * argv[])136 main(int argc, char* argv[]) {
137 #if !UCONFIG_NO_IDNA
138 char* filename = NULL;
139 #endif
140 const char *srcDir=NULL, *destDir=NULL, *icuUniDataDir=NULL;
141 const char *bundleName=NULL, *inputFileName = NULL;
142 char *basename=NULL;
143 int32_t sprepOptions = 0;
144
145 UErrorCode errorCode=U_ZERO_ERROR;
146
147 U_MAIN_INIT_ARGS(argc, argv);
148
149 /* preset then read command line options */
150 options[DESTDIR].value=u_getDataDirectory();
151 options[SOURCEDIR].value="";
152 options[UNICODE_VERSION].value="0"; /* don't assume the unicode version */
153 options[BUNDLE_NAME].value = DATA_NAME;
154 options[NORMALIZE].value = "";
155
156 argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
157
158 /* error handling, printing usage message */
159 if(argc<0) {
160 fprintf(stderr,
161 "error in command line argument \"%s\"\n",
162 argv[-argc]);
163 }
164 if(argc<0 || options[HELP].doesOccur || options[HELP_QUESTION_MARK].doesOccur) {
165 return printHelp(argc, argv);
166
167 }
168
169 /* get the options values */
170 beVerbose=options[VERBOSE].doesOccur;
171 haveCopyright=options[COPYRIGHT].doesOccur;
172 srcDir=options[SOURCEDIR].value;
173 destDir=options[DESTDIR].value;
174 bundleName = options[BUNDLE_NAME].value;
175 if(options[NORMALIZE].doesOccur) {
176 icuUniDataDir = options[NORMALIZE].value;
177 } else {
178 icuUniDataDir = options[NORM_CORRECTION_DIR].value;
179 }
180
181 if(argc<2) {
182 /* print the help message */
183 return printHelp(argc, argv);
184 } else {
185 inputFileName = argv[1];
186 }
187 if(!options[UNICODE_VERSION].doesOccur){
188 return printHelp(argc, argv);
189 }
190 if(options[ICUDATADIR].doesOccur) {
191 u_setDataDirectory(options[ICUDATADIR].value);
192 }
193 #if UCONFIG_NO_IDNA
194
195 fprintf(stderr,
196 "gensprep writes dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE
197 " because UCONFIG_NO_IDNA is set, \n"
198 "see icu/source/common/unicode/uconfig.h\n");
199 generateData(destDir, bundleName);
200
201 #else
202
203 setUnicodeVersion(options[UNICODE_VERSION].value);
204 filename = (char* ) uprv_malloc(uprv_strlen(srcDir) + 300); /* hopefully this should be enough */
205
206 /* prepare the filename beginning with the source dir */
207 if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL && uprv_strchr(srcDir,U_FILE_ALT_SEP_CHAR) == NULL){
208 filename[0] = '.';
209 filename[1] = U_FILE_SEP_CHAR;
210 uprv_strcpy(filename+2,srcDir);
211 }else{
212 uprv_strcpy(filename, srcDir);
213 }
214
215 basename=filename+uprv_strlen(filename);
216 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
217 *basename++=U_FILE_SEP_CHAR;
218 }
219
220 /* initialize */
221 init();
222
223 /* process the file */
224 uprv_strcpy(basename,inputFileName);
225 parseMappings(filename,FALSE, &errorCode);
226 if(U_FAILURE(errorCode)) {
227 fprintf(stderr, "Could not open file %s for reading. Error: %s \n", filename, u_errorName(errorCode));
228 return errorCode;
229 }
230
231 if(options[NORMALIZE].doesOccur){ /* this option might be set by @normalize;; in the source file */
232 /* set up directory for NormalizationCorrections.txt */
233 uprv_strcpy(filename,icuUniDataDir);
234 basename=filename+uprv_strlen(filename);
235 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
236 *basename++=U_FILE_SEP_CHAR;
237 }
238
239 *basename++=U_FILE_SEP_CHAR;
240 uprv_strcpy(basename,NORM_CORRECTIONS_FILE_NAME);
241
242 parseNormalizationCorrections(filename,&errorCode);
243 if(U_FAILURE(errorCode)){
244 fprintf(stderr,"Could not open file %s for reading \n", filename);
245 return errorCode;
246 }
247 sprepOptions |= _SPREP_NORMALIZATION_ON;
248 }
249
250 if(options[CHECK_BIDI].doesOccur){ /* this option might be set by @check-bidi;; in the source file */
251 sprepOptions |= _SPREP_CHECK_BIDI_ON;
252 }
253
254 setOptions(sprepOptions);
255
256 /* process parsed data */
257 if(U_SUCCESS(errorCode)) {
258 /* write the data file */
259 generateData(destDir, bundleName);
260
261 cleanUpData();
262 }
263
264 uprv_free(filename);
265
266 u_cleanup();
267
268 #endif
269
270 return errorCode;
271 }
272
273 #if !UCONFIG_NO_IDNA
274
275 static void U_CALLCONV
normalizationCorrectionsLineFn(void * context,char * fields[][2],int32_t fieldCount,UErrorCode * pErrorCode)276 normalizationCorrectionsLineFn(void *context,
277 char *fields[][2], int32_t fieldCount,
278 UErrorCode *pErrorCode) {
279 uint32_t mapping[40];
280 char *end, *s;
281 uint32_t code;
282 int32_t length;
283 UVersionInfo version;
284 UVersionInfo thisVersion;
285
286 /* get the character code, field 0 */
287 code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
288 if(U_FAILURE(*pErrorCode)) {
289 fprintf(stderr, "gensprep: error parsing NormalizationCorrections.txt mapping at %s\n", fields[0][0]);
290 exit(*pErrorCode);
291 }
292 /* Original (erroneous) decomposition */
293 s = fields[1][0];
294
295 /* parse the mapping string */
296 length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode);
297
298 /* ignore corrected decomposition */
299
300 u_versionFromString(version,fields[3][0] );
301 u_versionFromString(thisVersion, "3.2.0");
302
303
304
305 if(U_FAILURE(*pErrorCode)) {
306 fprintf(stderr, "gensprep error parsing NormalizationCorrections.txt of U+%04lx - %s\n",
307 (long)code, u_errorName(*pErrorCode));
308 exit(*pErrorCode);
309 }
310
311 /* store the mapping */
312 if( version[0] > thisVersion[0] ||
313 ((version[0]==thisVersion[0]) && (version[1] > thisVersion[1]))
314 ){
315 storeMapping(code,mapping, length, USPREP_MAP, pErrorCode);
316 }
317 setUnicodeVersionNC(version);
318 }
319
320 static void
parseNormalizationCorrections(const char * filename,UErrorCode * pErrorCode)321 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode) {
322 char *fields[4][2];
323
324 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
325 return;
326 }
327
328 u_parseDelimitedFile(filename, ';', fields, 4, normalizationCorrectionsLineFn, NULL, pErrorCode);
329
330 /* fprintf(stdout,"Number of code points that have NormalizationCorrections mapping with length >1 : %i\n",len); */
331
332 if(U_FAILURE(*pErrorCode) && ( *pErrorCode!=U_FILE_ACCESS_ERROR)) {
333 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
334 exit(*pErrorCode);
335 }
336 }
337
338 static void U_CALLCONV
strprepProfileLineFn(void * context,char * fields[][2],int32_t fieldCount,UErrorCode * pErrorCode)339 strprepProfileLineFn(void *context,
340 char *fields[][2], int32_t fieldCount,
341 UErrorCode *pErrorCode) {
342 uint32_t mapping[40];
343 char *end, *map;
344 uint32_t code;
345 int32_t length;
346 /*UBool* mapWithNorm = (UBool*) context;*/
347 const char* typeName;
348 uint32_t rangeStart=0,rangeEnd =0;
349 const char* filename = (const char*) context;
350 const char *s;
351
352 s = u_skipWhitespace(fields[0][0]);
353 if (*s == '@') {
354 /* special directive */
355 s++;
356 length = fields[0][1] - s;
357 if (length >= NORMALIZE_DIRECTIVE_LEN
358 && uprv_strncmp(s, NORMALIZE_DIRECTIVE, NORMALIZE_DIRECTIVE_LEN) == 0) {
359 options[NORMALIZE].doesOccur = TRUE;
360 return;
361 }
362 else if (length >= CHECK_BIDI_DIRECTIVE_LEN
363 && uprv_strncmp(s, CHECK_BIDI_DIRECTIVE, CHECK_BIDI_DIRECTIVE_LEN) == 0) {
364 options[CHECK_BIDI].doesOccur = TRUE;
365 return;
366 }
367 else {
368 fprintf(stderr, "gensprep error parsing a directive %s.", fields[0][0]);
369 }
370 }
371
372 typeName = fields[2][0];
373 map = fields[1][0];
374
375 if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){
376
377 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
378 if(U_FAILURE(*pErrorCode)){
379 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
380 return;
381 }
382
383 /* store the range */
384 storeRange(rangeStart,rangeEnd,USPREP_UNASSIGNED, pErrorCode);
385
386 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){
387
388 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
389 if(U_FAILURE(*pErrorCode)){
390 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
391 return;
392 }
393
394 /* store the range */
395 storeRange(rangeStart,rangeEnd,USPREP_PROHIBITED, pErrorCode);
396
397 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){
398
399 /* get the character code, field 0 */
400 code=(uint32_t)uprv_strtoul(s, &end, 16);
401 if(end<=s || end!=fields[0][1]) {
402 fprintf(stderr, "gensprep: syntax error in field 0 at %s\n", fields[0][0]);
403 *pErrorCode=U_PARSE_ERROR;
404 exit(U_PARSE_ERROR);
405 }
406
407 /* parse the mapping string */
408 length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode);
409
410 /* store the mapping */
411 storeMapping(code,mapping, length,USPREP_MAP, pErrorCode);
412
413 }else{
414 *pErrorCode = U_INVALID_FORMAT_ERROR;
415 }
416
417 if(U_FAILURE(*pErrorCode)) {
418 fprintf(stderr, "gensprep error parsing %s line %s at %s. Error: %s\n",filename,
419 fields[0][0],fields[2][0],u_errorName(*pErrorCode));
420 exit(*pErrorCode);
421 }
422
423 }
424
425 static void
parseMappings(const char * filename,UBool reportError,UErrorCode * pErrorCode)426 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode) {
427 char *fields[3][2];
428
429 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
430 return;
431 }
432
433 u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode);
434
435 /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/
436
437 if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
438 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
439 exit(*pErrorCode);
440 }
441 }
442
443
444 #endif /* #if !UCONFIG_NO_IDNA */
445
446 /*
447 * Hey, Emacs, please set the following:
448 *
449 * Local Variables:
450 * indent-tabs-mode: nil
451 * End:
452 *
453 */
454