1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 2003-2009, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: testidn.cpp
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 2003-02-06
14 * created by: Ram Viswanadha
15 *
16 * This program reads the rfc3454_*.txt files,
17 * parses them, and extracts the data for Nameprep conformance.
18 * It then preprocesses it and writes a binary file for efficient use
19 * in various IDNA conversion processes.
20 */
21
22 #include "unicode/utypes.h"
23
24 #if !UCONFIG_NO_IDNA && !UCONFIG_NO_TRANSLITERATION
25
26 #define USPREP_TYPE_NAMES_ARRAY
27
28 #include "unicode/uchar.h"
29 #include "unicode/putil.h"
30 #include "cmemory.h"
31 #include "cstring.h"
32 #include "unicode/udata.h"
33 #include "unewdata.h"
34 #include "uoptions.h"
35 #include "uparse.h"
36 #include "utrie.h"
37 #include "umutex.h"
38 #include "sprpimpl.h"
39 #include "testidna.h"
40 #include "punyref.h"
41 #include <stdlib.h>
42
43 UBool beVerbose=FALSE, haveCopyright=TRUE;
44
45 /* prototypes --------------------------------------------------------------- */
46
47
48 static void
49 parseMappings(const char *filename, UBool reportError,TestIDNA& test, UErrorCode *pErrorCode);
50
51 static void
52 compareMapping(uint32_t codepoint, uint32_t* mapping, int32_t mapLength,
53 UStringPrepType option);
54
55 static void
56 compareFlagsForRange(uint32_t start, uint32_t end,UStringPrepType option);
57
58 static void
59 testAllCodepoints(TestIDNA& test);
60
61 static TestIDNA* pTestIDNA =NULL;
62
63 static const char* fileNames[] = {
64 "rfc3491.txt"
65 };
66 static const UTrie *idnTrie = NULL;
67 static const int32_t *indexes = NULL;
68 static const uint16_t *mappingData = NULL;
69 /* -------------------------------------------------------------------------- */
70
71 /* file definitions */
72 #define DATA_TYPE "icu"
73
74 #define SPREP_DIR "sprep"
75
76 extern int
testData(TestIDNA & test)77 testData(TestIDNA& test) {
78 char *basename=NULL;
79 UErrorCode errorCode=U_ZERO_ERROR;
80 char *saveBasename =NULL;
81
82 LocalUStringPrepProfilePointer profile(usprep_openByType(USPREP_RFC3491_NAMEPREP, &errorCode));
83 if(U_FAILURE(errorCode)){
84 test.errcheckln(errorCode, "Failed to load IDNA data file. " + UnicodeString(u_errorName(errorCode)));
85 return errorCode;
86 }
87
88 char* filename = (char*) malloc(strlen(IntlTest::pathToDataDirectory())*1024);
89 //TODO get the srcDir dynamically
90 const char *srcDir=IntlTest::pathToDataDirectory();
91
92 idnTrie = &profile->sprepTrie;
93 indexes = profile->indexes;
94 mappingData = profile->mappingData;
95
96 //initialize
97 pTestIDNA = &test;
98
99 /* prepare the filename beginning with the source dir */
100 if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL){
101 filename[0] = 0x2E;
102 filename[1] = U_FILE_SEP_CHAR;
103 uprv_strcpy(filename+2,srcDir);
104 }else{
105 uprv_strcpy(filename, srcDir);
106 }
107 basename=filename+uprv_strlen(filename);
108 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
109 *basename++=U_FILE_SEP_CHAR;
110 }
111
112 /* process unassigned */
113 basename=filename+uprv_strlen(filename);
114 if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
115 *basename++=U_FILE_SEP_CHAR;
116 }
117
118 /* first copy misc directory */
119 saveBasename = basename;
120 uprv_strcpy(basename,SPREP_DIR);
121 basename = basename + uprv_strlen(SPREP_DIR);
122 *basename++=U_FILE_SEP_CHAR;
123
124 /* process unassigned */
125 uprv_strcpy(basename,fileNames[0]);
126 parseMappings(filename,TRUE, test,&errorCode);
127 if(U_FAILURE(errorCode)) {
128 test.errln( "Could not open file %s for reading \n", filename);
129 return errorCode;
130 }
131
132 testAllCodepoints(test);
133
134 pTestIDNA = NULL;
135 free(filename);
136 return errorCode;
137 }
138 U_CDECL_BEGIN
139
140 static void U_CALLCONV
strprepProfileLineFn(void *,char * fields[][2],int32_t fieldCount,UErrorCode * pErrorCode)141 strprepProfileLineFn(void * /*context*/,
142 char *fields[][2], int32_t fieldCount,
143 UErrorCode *pErrorCode) {
144 uint32_t mapping[40];
145 char *end, *map;
146 uint32_t code;
147 int32_t length;
148 /*UBool* mapWithNorm = (UBool*) context;*/
149 const char* typeName;
150 uint32_t rangeStart=0,rangeEnd =0;
151 const char *s;
152
153 s = u_skipWhitespace(fields[0][0]);
154 if (*s == '@') {
155 /* a special directive introduced in 4.2 */
156 return;
157 }
158
159 if(fieldCount != 3){
160 *pErrorCode = U_INVALID_FORMAT_ERROR;
161 return;
162 }
163
164 typeName = fields[2][0];
165 map = fields[1][0];
166
167 if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){
168
169 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
170
171 /* store the range */
172 compareFlagsForRange(rangeStart,rangeEnd,USPREP_UNASSIGNED);
173
174 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){
175
176 u_parseCodePointRange(s, &rangeStart,&rangeEnd, pErrorCode);
177
178 /* store the range */
179 compareFlagsForRange(rangeStart,rangeEnd,USPREP_PROHIBITED);
180
181 }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){
182 /* get the character code, field 0 */
183 code=(uint32_t)uprv_strtoul(s, &end, 16);
184
185 /* parse the mapping string */
186 length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode);
187
188 /* store the mapping */
189 compareMapping(code,mapping, length,USPREP_MAP);
190
191 }else{
192 *pErrorCode = U_INVALID_FORMAT_ERROR;
193 }
194
195 }
196
197 U_CDECL_END
198
199 static void
parseMappings(const char * filename,UBool reportError,TestIDNA & test,UErrorCode * pErrorCode)200 parseMappings(const char *filename,UBool reportError, TestIDNA& test, UErrorCode *pErrorCode) {
201 char *fields[3][2];
202
203 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
204 return;
205 }
206
207 u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode);
208
209 //fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);
210
211 if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
212 test.errln( "testidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
213 }
214 }
215
216
217 static inline UStringPrepType
getValues(uint32_t result,int32_t & value,UBool & isIndex)218 getValues(uint32_t result, int32_t& value, UBool& isIndex){
219
220 UStringPrepType type;
221
222 if(result == 0){
223 /*
224 * Initial value stored in the mapping table
225 * just return USPREP_TYPE_LIMIT .. so that
226 * the source codepoint is copied to the destination
227 */
228 type = USPREP_TYPE_LIMIT;
229 isIndex =FALSE;
230 value = 0;
231 }else if(result >= _SPREP_TYPE_THRESHOLD){
232 type = (UStringPrepType) (result - _SPREP_TYPE_THRESHOLD);
233 isIndex =FALSE;
234 value = 0;
235 }else{
236 /* get the state */
237 type = USPREP_MAP;
238 /* ascertain if the value is index or delta */
239 if(result & 0x02){
240 isIndex = TRUE;
241 value = result >> 2; //mask off the lower 2 bits and shift
242
243 }else{
244 isIndex = FALSE;
245 value = (int16_t)result;
246 value = (value >> 2);
247
248 }
249 if((result>>2) == _SPREP_MAX_INDEX_VALUE){
250 type = USPREP_DELETE;
251 isIndex =FALSE;
252 value = 0;
253 }
254 }
255 return type;
256 }
257
258
259
260 static void
testAllCodepoints(TestIDNA & test)261 testAllCodepoints(TestIDNA& test){
262 /*
263 {
264 UChar str[19] = {
265 0xC138, 0xACC4, 0xC758, 0xBAA8, 0xB4E0, 0xC0AC, 0xB78C, 0xB4E4, 0xC774,
266 0x070F,//prohibited
267 0xD55C, 0xAD6D, 0xC5B4, 0xB97C, 0xC774, 0xD574, 0xD55C, 0xB2E4, 0xBA74
268 };
269 uint32_t in[19] = {0};
270 UErrorCode status = U_ZERO_ERROR;
271 int32_t inLength=0, outLength=100;
272 char output[100] = {0};
273 punycode_status error;
274 u_strToUTF32((UChar32*)in,19,&inLength,str,19,&status);
275
276 error= punycode_encode(inLength, in, NULL, (uint32_t*)&outLength, output);
277 printf(output);
278
279 }
280 */
281
282 uint32_t i = 0;
283 int32_t unassigned = 0;
284 int32_t prohibited = 0;
285 int32_t mappedWithNorm = 0;
286 int32_t mapped = 0;
287 int32_t noValueInTrie = 0;
288
289 UStringPrepType type;
290 int32_t value;
291 UBool isIndex = FALSE;
292
293 for(i=0;i<=0x10FFFF;i++){
294 uint32_t result = 0;
295 UTRIE_GET16(idnTrie,i, result);
296 type = getValues(result,value, isIndex);
297 if(type != USPREP_TYPE_LIMIT ){
298 if(type == USPREP_UNASSIGNED){
299 unassigned++;
300 }
301 if(type == USPREP_PROHIBITED){
302 prohibited++;
303 }
304 if(type == USPREP_MAP){
305 mapped++;
306 }
307 }else{
308 noValueInTrie++;
309 if(result > 0){
310 test.errln("The return value for 0x%06X is wrong. %i\n",i,result);
311 }
312 }
313 }
314
315 test.logln("Number of Unassinged code points : %i \n",unassigned);
316 test.logln("Number of Prohibited code points : %i \n",prohibited);
317 test.logln("Number of Mapped code points : %i \n",mapped);
318 test.logln("Number of Mapped with NFKC code points : %i \n",mappedWithNorm);
319 test.logln("Number of code points that have no value in Trie: %i \n",noValueInTrie);
320
321
322 }
323
324 static void
compareMapping(uint32_t codepoint,uint32_t * mapping,int32_t mapLength,UStringPrepType type)325 compareMapping(uint32_t codepoint, uint32_t* mapping,int32_t mapLength,
326 UStringPrepType type){
327 uint32_t result = 0;
328 UTRIE_GET16(idnTrie,codepoint, result);
329
330 int32_t length=0;
331 UBool isIndex;
332 UStringPrepType retType;
333 int32_t value, index=0, delta=0;
334
335 retType = getValues(result,value,isIndex);
336
337
338 if(type != retType && retType != USPREP_DELETE){
339
340 pTestIDNA->errln( "Did not get the assigned type for codepoint 0x%08X. Expected: %i Got: %i\n",codepoint, USPREP_MAP, type);
341
342 }
343
344 if(isIndex){
345 index = value;
346 if(index >= indexes[_SPREP_ONE_UCHAR_MAPPING_INDEX_START] &&
347 index < indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START]){
348 length = 1;
349 }else if(index >= indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START] &&
350 index < indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START]){
351 length = 2;
352 }else if(index >= indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START] &&
353 index < indexes[_SPREP_FOUR_UCHARS_MAPPING_INDEX_START]){
354 length = 3;
355 }else{
356 length = mappingData[index++];
357 }
358 }else{
359 delta = value;
360 length = (retType == USPREP_DELETE)? 0 : 1;
361 }
362
363 int32_t realLength =0;
364 /* figure out the real length */
365 for(int32_t j=0; j<mapLength; j++){
366 if(mapping[j] > 0xFFFF){
367 realLength +=2;
368 }else{
369 realLength++;
370 }
371 }
372
373 if(realLength != length){
374 pTestIDNA->errln( "Did not get the expected length. Expected: %i Got: %i\n", mapLength, length);
375 }
376
377 if(isIndex){
378 for(int8_t i =0; i< mapLength; i++){
379 if(mapping[i] <= 0xFFFF){
380 if(mappingData[index+i] != (uint16_t)mapping[i]){
381 pTestIDNA->errln("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[i], mappingData[index+i]);
382 }
383 }else{
384 UChar lead = UTF16_LEAD(mapping[i]);
385 UChar trail = UTF16_TRAIL(mapping[i]);
386 if(mappingData[index+i] != lead ||
387 mappingData[index+i+1] != trail){
388 pTestIDNA->errln( "Did not get the expected result. Expected: 0x%04X 0x%04X Got: 0x%04X 0x%04X", lead, trail, mappingData[index+i], mappingData[index+i+1]);
389 }
390 }
391 }
392 }else{
393 if(retType!=USPREP_DELETE && (codepoint-delta) != (uint16_t)mapping[0]){
394 pTestIDNA->errln("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[0],(codepoint-delta));
395 }
396 }
397
398 }
399
400 static void
compareFlagsForRange(uint32_t start,uint32_t end,UStringPrepType type)401 compareFlagsForRange(uint32_t start, uint32_t end,
402 UStringPrepType type){
403
404 uint32_t result =0 ;
405 UStringPrepType retType;
406 UBool isIndex=FALSE;
407 int32_t value=0;
408 /*
409 // supplementary code point
410 UChar __lead16=UTF16_LEAD(0x2323E);
411 int32_t __offset;
412
413 // get data for lead surrogate
414 (result)=_UTRIE_GET_RAW((&idnTrie), index, 0, (__lead16));
415 __offset=(&idnTrie)->getFoldingOffset(result);
416
417 // get the real data from the folded lead/trail units
418 if(__offset>0) {
419 (result)=_UTRIE_GET_RAW((&idnTrie), index, __offset, (0x2323E)&0x3ff);
420 } else {
421 (result)=(uint32_t)((&idnTrie)->initialValue);
422 }
423
424 UTRIE_GET16(&idnTrie,0x2323E, result);
425 */
426 while(start < end+1){
427 UTRIE_GET16(idnTrie,start, result);
428 retType = getValues(result,value,isIndex);
429 if(result > _SPREP_TYPE_THRESHOLD){
430 if(retType != type){
431 pTestIDNA->errln( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]);
432 }
433 }else{
434 if(type == USPREP_PROHIBITED && ((result & 0x01) != 0x01)){
435 pTestIDNA->errln( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]);
436 }
437 }
438
439 start++;
440 }
441
442 }
443
444
445 #endif /* #if !UCONFIG_NO_IDNA */
446
447 /*
448 * Hey, Emacs, please set the following:
449 *
450 * Local Variables:
451 * indent-tabs-mode: nil
452 * End:
453 *
454 */
455