1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 2003-2006, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: gensprep.c
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 2003-02-06
14 * created by: Ram Viswanadha
15 *
16 * This program reads the Profile.txt files,
17 * parses them, and extracts the data for StringPrep profile.
18 * It then preprocesses it and writes a binary file for efficient use
19 * in various StringPrep conversion processes.
20 */
21
22 #define USPREP_TYPE_NAMES_ARRAY 1
23
24 #include <stdio.h>
25 #include <stdlib.h>
26
27 #include "cmemory.h"
28 #include "cstring.h"
29 #include "unewdata.h"
30 #include "uoptions.h"
31 #include "uparse.h"
32 #include "sprpimpl.h"
33
34 #include "unicode/udata.h"
35 #include "unicode/utypes.h"
36 #include "unicode/putil.h"
37
38
39 U_CDECL_BEGIN
40 #include "gensprep.h"
41 U_CDECL_END
42
43 UBool beVerbose=FALSE, haveCopyright=TRUE;
44
45 #define NORM_CORRECTIONS_FILE_NAME "NormalizationCorrections.txt"
46
47 /* prototypes --------------------------------------------------------------- */
48
49 static void
50 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode);
51
52 static void
53 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode);
54
55
56 /* -------------------------------------------------------------------------- */
57
58 static UOption options[]={
59 UOPTION_HELP_H,
60 UOPTION_HELP_QUESTION_MARK,
61 UOPTION_VERBOSE,
62 UOPTION_COPYRIGHT,
63 UOPTION_DESTDIR,
64 UOPTION_SOURCEDIR,
65 UOPTION_ICUDATADIR,
66 UOPTION_BUNDLE_NAME,
67 { "normalization", NULL, NULL, NULL, 'n', UOPT_REQUIRES_ARG, 0 },
68 { "check-bidi", NULL, NULL, NULL, 'k', UOPT_NO_ARG, 0},
69 { "unicode", NULL, NULL, NULL, 'u', UOPT_REQUIRES_ARG, 0 },
70 };
71
72 enum{
73 HELP,
74 HELP_QUESTION_MARK,
75 VERBOSE,
76 COPYRIGHT,
77 DESTDIR,
78 SOURCEDIR,
79 ICUDATADIR,
80 BUNDLE_NAME,
81 NORMALIZE,
82 CHECK_BIDI,
83 UNICODE_VERSION
84 };
85
printHelp(int argc,char * argv[])86 static int printHelp(int argc, char* argv[]){
87 /*
88 * Broken into chucks because the C89 standard says the minimum
89 * required supported string length is 509 bytes.
90 */
91 fprintf(stderr,
92 "Usage: %s [-options] [file_name]\n"
93 "\n"
94 "Read the files specified and\n"
95 "create a binary file [package-name]_[bundle-name]." DATA_TYPE " with the StringPrep profile data\n"
96 "\n",
97 argv[0]);
98 fprintf(stderr,
99 "Options:\n"
100 "\t-h or -? or --help print this usage text\n"
101 "\t-v or --verbose verbose output\n"
102 "\t-c or --copyright include a copyright notice\n");
103 fprintf(stderr,
104 "\t-d or --destdir destination directory, followed by the path\n"
105 "\t-s or --sourcedir source directory of ICU data, followed by the path\n"
106 "\t-b or --bundle-name generate the ouput data file with the name specified\n"
107 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n"
108 "\t followed by path, defaults to %s\n",
109 u_getDataDirectory());
110 fprintf(stderr,
111 "\t-n or --normalize turn on the option for normalization and include mappings\n"
112 "\t from NormalizationCorrections.txt from the given path,\n"
113 "\t e.g: /test/icu/source/data/unidata\n"
114 "\t-k or --check-bidi turn on the option for checking for BiDi in the profile\n"
115 "\t-u or --unicode version of Unicode to be used with this profile followed by the version\n"
116 );
117 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
118 }
119
120
121 extern int
main(int argc,char * argv[])122 main(int argc, char* argv[]) {
123 #if !UCONFIG_NO_IDNA
124 char* filename = NULL;
125 #endif
126 const char *srcDir=NULL, *destDir=NULL, *icuUniDataDir=NULL;
127 const char *bundleName=NULL, *inputFileName = NULL;
128 char *basename=NULL;
129 int32_t sprepOptions = 0;
130
131 UErrorCode errorCode=U_ZERO_ERROR;
132
133 U_MAIN_INIT_ARGS(argc, argv);
134
135 /* preset then read command line options */
136 options[DESTDIR].value=u_getDataDirectory();
137 options[SOURCEDIR].value="";
138 options[UNICODE_VERSION].value="0"; /* don't assume the unicode version */
139 options[BUNDLE_NAME].value = DATA_NAME;
140 options[NORMALIZE].value = "";
141
142 argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
143
144 /* error handling, printing usage message */
145 if(argc<0) {
146 fprintf(stderr,
147 "error in command line argument \"%s\"\n",
148 argv[-argc]);
149 }
150 if(argc<0 || options[HELP].doesOccur || options[HELP_QUESTION_MARK].doesOccur) {
151 return printHelp(argc, argv);
152
153 }
154
155 /* get the options values */
156 beVerbose=options[VERBOSE].doesOccur;
157 haveCopyright=options[COPYRIGHT].doesOccur;
158 srcDir=options[SOURCEDIR].value;
159 destDir=options[DESTDIR].value;
160 bundleName = options[BUNDLE_NAME].value;
161 icuUniDataDir = options[NORMALIZE].value;
162
163 if(argc<2) {
164 /* print the help message */
165 return printHelp(argc, argv);
166 } else {
167 inputFileName = argv[1];
168 }
169 if(!options[UNICODE_VERSION].doesOccur){
170 return printHelp(argc, argv);
171 }
172 if(options[ICUDATADIR].doesOccur) {
173 u_setDataDirectory(options[ICUDATADIR].value);
174 }
175 #if UCONFIG_NO_IDNA
176
177 fprintf(stderr,
178 "gensprep writes dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE
179 " because UCONFIG_NO_IDNA is set, \n"
180 "see icu/source/common/unicode/uconfig.h\n");
181 generateData(destDir, bundleName);
182
183 #else
184
185 setUnicodeVersion(options[UNICODE_VERSION].value);
186 filename = (char* ) uprv_malloc(uprv_strlen(srcDir) + 300); /* hopefully this should be enough */
187
188 /* prepare the filename beginning with the source dir */
189 if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL && uprv_strchr(srcDir,U_FILE_ALT_SEP_CHAR) == NULL){
190 filename[0] = '.';
191 filename[1] = U_FILE_SEP_CHAR;
192 uprv_strcpy(filename+2,srcDir);
193 }else{
194 uprv_strcpy(filename, srcDir);
195 }
196
197 basename=filename+uprv_strlen(filename);
198 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
199 *basename++=U_FILE_SEP_CHAR;
200 }
201
202 /* initialize */
203 init();
204
205 /* process the file */
206 uprv_strcpy(basename,inputFileName);
207 parseMappings(filename,FALSE, &errorCode);
208 if(U_FAILURE(errorCode)) {
209 fprintf(stderr, "Could not open file %s for reading. Error: %s \n", filename, u_errorName(errorCode));
210 return errorCode;
211 }
212
213 if(options[NORMALIZE].doesOccur){
214 /* set up directory for NormalizationCorrections.txt */
215 uprv_strcpy(filename,icuUniDataDir);
216 basename=filename+uprv_strlen(filename);
217 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
218 *basename++=U_FILE_SEP_CHAR;
219 }
220
221 *basename++=U_FILE_SEP_CHAR;
222 uprv_strcpy(basename,NORM_CORRECTIONS_FILE_NAME);
223
224 parseNormalizationCorrections(filename,&errorCode);
225 if(U_FAILURE(errorCode)){
226 fprintf(stderr,"Could not open file %s for reading \n", filename);
227 return errorCode;
228 }
229 sprepOptions |= _SPREP_NORMALIZATION_ON;
230 }
231
232 if(options[CHECK_BIDI].doesOccur){
233 sprepOptions |= _SPREP_CHECK_BIDI_ON;
234 }
235
236 setOptions(sprepOptions);
237
238 /* process parsed data */
239 if(U_SUCCESS(errorCode)) {
240 /* write the data file */
241 generateData(destDir, bundleName);
242
243 cleanUpData();
244 }
245
246 uprv_free(filename);
247
248 #endif
249
250 return errorCode;
251 }
252
253 #if !UCONFIG_NO_IDNA
254
255 static void U_CALLCONV
normalizationCorrectionsLineFn(void * context,char * fields[][2],int32_t fieldCount,UErrorCode * pErrorCode)256 normalizationCorrectionsLineFn(void *context,
257 char *fields[][2], int32_t fieldCount,
258 UErrorCode *pErrorCode) {
259 uint32_t mapping[40];
260 char *end, *s;
261 uint32_t code;
262 int32_t length;
263 UVersionInfo version;
264 UVersionInfo thisVersion;
265
266 /* get the character code, field 0 */
267 code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
268 if(U_FAILURE(*pErrorCode)) {
269 fprintf(stderr, "gensprep: error parsing NormalizationCorrections.txt mapping at %s\n", fields[0][0]);
270 exit(*pErrorCode);
271 }
272 /* Original (erroneous) decomposition */
273 s = fields[1][0];
274
275 /* parse the mapping string */
276 length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode);
277
278 /* ignore corrected decomposition */
279
280 u_versionFromString(version,fields[3][0] );
281 u_versionFromString(thisVersion, "3.2.0");
282
283
284
285 if(U_FAILURE(*pErrorCode)) {
286 fprintf(stderr, "gensprep error parsing NormalizationCorrections.txt of U+%04lx - %s\n",
287 (long)code, u_errorName(*pErrorCode));
288 exit(*pErrorCode);
289 }
290
291 /* store the mapping */
292 if( version[0] > thisVersion[0] ||
293 ((version[0]==thisVersion[0]) && (version[1] > thisVersion[1]))
294 ){
295 storeMapping(code,mapping, length, USPREP_MAP, pErrorCode);
296 }
297 setUnicodeVersionNC(version);
298 }
299
300 static void
parseNormalizationCorrections(const char * filename,UErrorCode * pErrorCode)301 parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode) {
302 char *fields[4][2];
303
304 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
305 return;
306 }
307
308 u_parseDelimitedFile(filename, ';', fields, 4, normalizationCorrectionsLineFn, NULL, pErrorCode);
309
310 /* fprintf(stdout,"Number of code points that have NormalizationCorrections mapping with length >1 : %i\n",len); */
311
312 if(U_FAILURE(*pErrorCode) && ( *pErrorCode!=U_FILE_ACCESS_ERROR)) {
313 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
314 exit(*pErrorCode);
315 }
316 }
317
318 static void U_CALLCONV
strprepProfileLineFn(void * context,char * fields[][2],int32_t fieldCount,UErrorCode * pErrorCode)319 strprepProfileLineFn(void *context,
320 char *fields[][2], int32_t fieldCount,
321 UErrorCode *pErrorCode) {
322 uint32_t mapping[40];
323 char *end, *map;
324 uint32_t code;
325 int32_t length;
326 /*UBool* mapWithNorm = (UBool*) context;*/
327 const char* typeName;
328 uint32_t rangeStart=0,rangeEnd =0;
329 const char* filename = (const char*) context;
330
331 typeName = fields[2][0];
332 map = fields[1][0];
333
334 if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){
335
336 u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode);
337 if(U_FAILURE(*pErrorCode)){
338 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
339 return;
340 }
341
342 /* store the range */
343 storeRange(rangeStart,rangeEnd,USPREP_UNASSIGNED, pErrorCode);
344
345 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){
346
347 u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode);
348 if(U_FAILURE(*pErrorCode)){
349 fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
350 return;
351 }
352
353 /* store the range */
354 storeRange(rangeStart,rangeEnd,USPREP_PROHIBITED, pErrorCode);
355
356 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){
357
358 /* get the character code, field 0 */
359 code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
360 if(end<=fields[0][0] || end!=fields[0][1]) {
361 fprintf(stderr, "gensprep: syntax error in field 0 at %s\n", fields[0][0]);
362 *pErrorCode=U_PARSE_ERROR;
363 exit(U_PARSE_ERROR);
364 }
365
366 /* parse the mapping string */
367 length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode);
368
369 /* store the mapping */
370 storeMapping(code,mapping, length,USPREP_MAP, pErrorCode);
371
372 }else{
373 *pErrorCode = U_INVALID_FORMAT_ERROR;
374 }
375
376 if(U_FAILURE(*pErrorCode)) {
377 fprintf(stderr, "gensprep error parsing %s line %s at %s. Error: %s\n",filename,
378 fields[0][0],fields[2][0],u_errorName(*pErrorCode));
379 exit(*pErrorCode);
380 }
381
382 }
383
384 static void
parseMappings(const char * filename,UBool reportError,UErrorCode * pErrorCode)385 parseMappings(const char *filename, UBool reportError, UErrorCode *pErrorCode) {
386 char *fields[3][2];
387
388 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
389 return;
390 }
391
392 u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode);
393
394 /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/
395
396 if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
397 fprintf(stderr, "gensprep error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
398 exit(*pErrorCode);
399 }
400 }
401
402
403 #endif /* #if !UCONFIG_NO_IDNA */
404
405 /*
406 * Hey, Emacs, please set the following:
407 *
408 * Local Variables:
409 * indent-tabs-mode: nil
410 * End:
411 *
412 */
413