1 /*
2 ************************************************************************
3 * Copyright (c) 1997-2004, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 ************************************************************************
6 */
7
8 #include "unicode/utypes.h"
9
10 #if !UCONFIG_NO_NORMALIZATION
11
12 #include "unicode/uchar.h"
13 #include "unicode/normlzr.h"
14 #include "unicode/uniset.h"
15 #include "unicode/putil.h"
16 #include "unormimp.h"
17 #include "cstring.h"
18 #include "filestrm.h"
19 #include "normconf.h"
20 #include <stdio.h>
21
22 #define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
23
24 #define CASE(id,test) case id: \
25 name = #test; \
26 if (exec) { \
27 logln(#test "---"); \
28 logln((UnicodeString)""); \
29 test(); \
30 } \
31 break
32
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)33 void NormalizerConformanceTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* /*par*/) {
34 switch (index) {
35 CASE(0, TestConformance);
36 CASE(1, TestConformance32);
37 // CASE(2, TestCase6);
38 default: name = ""; break;
39 }
40 }
41
42 #define FIELD_COUNT 5
43
NormalizerConformanceTest()44 NormalizerConformanceTest::NormalizerConformanceTest() :
45 normalizer(UnicodeString(), UNORM_NFC) {}
46
~NormalizerConformanceTest()47 NormalizerConformanceTest::~NormalizerConformanceTest() {}
48
49 // more interesting conformance test cases, not in the unicode.org NormalizationTest.txt
50 static const char *moreCases[]={
51 // Markus 2001aug30
52 "0061 0332 0308;00E4 0332;0061 0332 0308;00E4 0332;0061 0332 0308; # Markus 0",
53
54 // Markus 2001oct26 - test edge case for iteration: U+0f73.cc==0 but decomposition.lead.cc==129
55 "0061 0301 0F73;00E1 0F71 0F72;0061 0F71 0F72 0301;00E1 0F71 0F72;0061 0F71 0F72 0301; # Markus 1"
56 };
57
compare(const UnicodeString & s1,const UnicodeString & s2)58 void NormalizerConformanceTest::compare(const UnicodeString& s1, const UnicodeString& s2){
59 UErrorCode status=U_ZERO_ERROR;
60 // TODO: Re-enable this tests after UTC fixes UAX 21
61 if(s1.indexOf((UChar32)0x0345)>=0)return;
62 if(Normalizer::compare(s1,s2,U_FOLD_CASE_DEFAULT,status)!=0){
63 errln("Normalizer::compare() failed for s1: " + prettify(s1) + " s2: " +prettify(s2));
64 }
65 }
66
67 FileStream *
openNormalizationTestFile(const char * filename)68 NormalizerConformanceTest::openNormalizationTestFile(const char *filename) {
69 char unidataPath[2000];
70 const char *folder;
71 FileStream *input;
72 UErrorCode errorCode;
73
74 // look inside ICU_DATA first
75 folder=pathToDataDirectory();
76 if(folder!=NULL) {
77 strcpy(unidataPath, folder);
78 strcat(unidataPath, "unidata" U_FILE_SEP_STRING);
79 strcat(unidataPath, filename);
80 input=T_FileStream_open(unidataPath, "rb");
81 if(input!=NULL) {
82 return input;
83 }
84 }
85
86 // find icu/source/data/unidata relative to the test data
87 errorCode=U_ZERO_ERROR;
88 folder=loadTestData(errorCode);
89 if(U_SUCCESS(errorCode)) {
90 strcpy(unidataPath, folder);
91 strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
92 U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
93 U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
94 strcat(unidataPath, filename);
95 input=T_FileStream_open(unidataPath, "rb");
96 if(input!=NULL) {
97 return input;
98 }
99 }
100
101 // look in icu/source/test/testdata/out/build
102 errorCode=U_ZERO_ERROR;
103 folder=loadTestData(errorCode);
104 if(U_SUCCESS(errorCode)) {
105 strcpy(unidataPath, folder);
106 strcat(unidataPath, U_FILE_SEP_STRING);
107 strcat(unidataPath, filename);
108 input=T_FileStream_open(unidataPath, "rb");
109 if(input!=NULL) {
110 return input;
111 }
112 }
113
114 // look in icu/source/test/testdata
115 errorCode=U_ZERO_ERROR;
116 folder=loadTestData(errorCode);
117 if(U_SUCCESS(errorCode)) {
118 strcpy(unidataPath, folder);
119 strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING);
120 strcat(unidataPath, filename);
121 input=T_FileStream_open(unidataPath, "rb");
122 if(input!=NULL) {
123 return input;
124 }
125 }
126
127 // find icu/source/data/unidata relative to U_TOPSRCDIR
128 #if defined(U_TOPSRCDIR)
129 strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
130 strcat(unidataPath, filename);
131 input=T_FileStream_open(unidataPath, "rb");
132 if(input!=NULL) {
133 return input;
134 }
135
136 strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "test" U_FILE_SEP_STRING "testdata" U_FILE_SEP_STRING);
137 strcat(unidataPath, filename);
138 input=T_FileStream_open(unidataPath, "rb");
139 if(input!=NULL) {
140 return input;
141 }
142 #endif
143
144 errln("Failed to open %s", filename);
145 return NULL;
146 }
147
148 /**
149 * Test the conformance of Normalizer to
150 * http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt
151 */
TestConformance()152 void NormalizerConformanceTest::TestConformance() {
153 TestConformance(openNormalizationTestFile("NormalizationTest.txt"), 0);
154 }
155
TestConformance32()156 void NormalizerConformanceTest::TestConformance32() {
157 TestConformance(openNormalizationTestFile("NormalizationTest-3.2.0.txt"), UNORM_UNICODE_3_2);
158 }
159
TestConformance(FileStream * input,int32_t options)160 void NormalizerConformanceTest::TestConformance(FileStream *input, int32_t options) {
161 enum { BUF_SIZE = 1024 };
162 char lineBuf[BUF_SIZE];
163 UnicodeString fields[FIELD_COUNT];
164 UErrorCode status = U_ZERO_ERROR;
165 int32_t passCount = 0;
166 int32_t failCount = 0;
167 UChar32 c;
168
169 if(input==NULL) {
170 return;
171 }
172
173 // UnicodeSet for all code points that are not mentioned in NormalizationTest.txt
174 UnicodeSet other(0, 0x10ffff);
175
176 int32_t count, countMoreCases = sizeof(moreCases)/sizeof(moreCases[0]);
177 for (count = 1;;++count) {
178 if (!T_FileStream_eof(input)) {
179 T_FileStream_readLine(input, lineBuf, (int32_t)sizeof(lineBuf));
180 } else {
181 // once NormalizationTest.txt is finished, use moreCases[]
182 if(count > countMoreCases) {
183 count = 0;
184 } else if(count == countMoreCases) {
185 // all done
186 break;
187 }
188 uprv_strcpy(lineBuf, moreCases[count]);
189 }
190 if (lineBuf[0] == 0 || lineBuf[0] == '\n' || lineBuf[0] == '\r') continue;
191
192 // Expect 5 columns of this format:
193 // 1E0C;1E0C;0044 0323;1E0C;0044 0323; # <comments>
194
195 // Parse out the comment.
196 if (lineBuf[0] == '#') continue;
197
198 // Read separator lines starting with '@'
199 if (lineBuf[0] == '@') {
200 logln(lineBuf);
201 continue;
202 }
203
204 // Parse out the fields
205 if (!hexsplit(lineBuf, ';', fields, FIELD_COUNT)) {
206 errln((UnicodeString)"Unable to parse line " + count);
207 break; // Syntax error
208 }
209
210 // Remove a single code point from the "other" UnicodeSet
211 if(fields[0].length()==fields[0].moveIndex32(0, 1)) {
212 c=fields[0].char32At(0);
213 if(0xac20<=c && c<=0xd73f && quick) {
214 // not an exhaustive test run: skip most Hangul syllables
215 if(c==0xac20) {
216 other.remove(0xac20, 0xd73f);
217 }
218 continue;
219 }
220 other.remove(c);
221 }
222
223 if (checkConformance(fields, lineBuf, options, status)) {
224 ++passCount;
225 } else {
226 ++failCount;
227 if(status == U_FILE_ACCESS_ERROR) {
228 errln("Something is wrong with the normalizer, skipping the rest of the test.");
229 break;
230 }
231 }
232 if ((count % 1000) == 0) {
233 logln("Line %d", count);
234 }
235 }
236
237 T_FileStream_close(input);
238
239 /*
240 * Test that all characters that are not mentioned
241 * as single code points in column 1
242 * do not change under any normalization.
243 */
244
245 // remove U+ffff because that is the end-of-iteration sentinel value
246 other.remove(0xffff);
247
248 for(c=0; c<=0x10ffff; quick ? c+=113 : ++c) {
249 if(0x30000<=c && c<0xe0000) {
250 c=0xe0000;
251 }
252 if(!other.contains(c)) {
253 continue;
254 }
255
256 fields[0]=fields[1]=fields[2]=fields[3]=fields[4].setTo(c);
257 sprintf(lineBuf, "not mentioned code point U+%04lx", (long)c);
258
259 if (checkConformance(fields, lineBuf, options, status)) {
260 ++passCount;
261 } else {
262 ++failCount;
263 if(status == U_FILE_ACCESS_ERROR) {
264 errln("Something is wrong with the normalizer, skipping the rest of the test.");
265 break;
266 }
267 }
268 if ((c % 0x1000) == 0) {
269 logln("Code point U+%04lx", c);
270 }
271 }
272
273 if (failCount != 0) {
274 errln((UnicodeString)"Total: " + failCount + " lines/code points failed, " +
275 passCount + " lines/code points passed");
276 } else {
277 logln((UnicodeString)"Total: " + passCount + " lines/code points passed");
278 }
279 }
280
281 /**
282 * Verify the conformance of the given line of the Unicode
283 * normalization (UTR 15) test suite file. For each line,
284 * there are five columns, corresponding to field[0]..field[4].
285 *
286 * The following invariants must be true for all conformant implementations
287 * c2 == NFC(c1) == NFC(c2) == NFC(c3)
288 * c3 == NFD(c1) == NFD(c2) == NFD(c3)
289 * c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
290 * c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
291 *
292 * @param field the 5 columns
293 * @param line the source line from the test suite file
294 * @return true if the test passes
295 */
checkConformance(const UnicodeString * field,const char * line,int32_t options,UErrorCode & status)296 UBool NormalizerConformanceTest::checkConformance(const UnicodeString* field,
297 const char *line,
298 int32_t options,
299 UErrorCode &status) {
300 UBool pass = TRUE, result;
301 //UErrorCode status = U_ZERO_ERROR;
302 UnicodeString out, fcd;
303 int32_t fieldNum;
304
305 for (int32_t i=0; i<FIELD_COUNT; ++i) {
306 fieldNum = i+1;
307 if (i<3) {
308 Normalizer::normalize(field[i], UNORM_NFC, options, out, status);
309 pass &= assertEqual("C", field[i], out, field[1], "c2!=C(c", fieldNum);
310 iterativeNorm(field[i], UNORM_NFC, options, out, +1);
311 pass &= assertEqual("C(+1)", field[i], out, field[1], "c2!=C(c", fieldNum);
312 iterativeNorm(field[i], UNORM_NFC, options, out, -1);
313 pass &= assertEqual("C(-1)", field[i], out, field[1], "c2!=C(c", fieldNum);
314
315 Normalizer::normalize(field[i], UNORM_NFD, options, out, status);
316 pass &= assertEqual("D", field[i], out, field[2], "c3!=D(c", fieldNum);
317 iterativeNorm(field[i], UNORM_NFD, options, out, +1);
318 pass &= assertEqual("D(+1)", field[i], out, field[2], "c3!=D(c", fieldNum);
319 iterativeNorm(field[i], UNORM_NFD, options, out, -1);
320 pass &= assertEqual("D(-1)", field[i], out, field[2], "c3!=D(c", fieldNum);
321 }
322 Normalizer::normalize(field[i], UNORM_NFKC, options, out, status);
323 pass &= assertEqual("KC", field[i], out, field[3], "c4!=KC(c", fieldNum);
324 iterativeNorm(field[i], UNORM_NFKC, options, out, +1);
325 pass &= assertEqual("KC(+1)", field[i], out, field[3], "c4!=KC(c", fieldNum);
326 iterativeNorm(field[i], UNORM_NFKC, options, out, -1);
327 pass &= assertEqual("KC(-1)", field[i], out, field[3], "c4!=KC(c", fieldNum);
328
329 Normalizer::normalize(field[i], UNORM_NFKD, options, out, status);
330 pass &= assertEqual("KD", field[i], out, field[4], "c5!=KD(c", fieldNum);
331 iterativeNorm(field[i], UNORM_NFKD, options, out, +1);
332 pass &= assertEqual("KD(+1)", field[i], out, field[4], "c5!=KD(c", fieldNum);
333 iterativeNorm(field[i], UNORM_NFKD, options, out, -1);
334 pass &= assertEqual("KD(-1)", field[i], out, field[4], "c5!=KD(c", fieldNum);
335 }
336 compare(field[1],field[2]);
337 compare(field[0],field[1]);
338 // test quick checks
339 if(UNORM_NO == Normalizer::quickCheck(field[1], UNORM_NFC, options, status)) {
340 errln("Normalizer error: quickCheck(NFC(s), UNORM_NFC) is UNORM_NO");
341 pass = FALSE;
342 }
343 if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_NFD, options, status)) {
344 errln("Normalizer error: quickCheck(NFD(s), UNORM_NFD) is UNORM_NO");
345 pass = FALSE;
346 }
347 if(UNORM_NO == Normalizer::quickCheck(field[3], UNORM_NFKC, options, status)) {
348 errln("Normalizer error: quickCheck(NFKC(s), UNORM_NFKC) is UNORM_NO");
349 pass = FALSE;
350 }
351 if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_NFKD, options, status)) {
352 errln("Normalizer error: quickCheck(NFKD(s), UNORM_NFKD) is UNORM_NO");
353 pass = FALSE;
354 }
355
356 // branch on options==0 for better code coverage
357 if(options==0) {
358 result = Normalizer::isNormalized(field[1], UNORM_NFC, status);
359 } else {
360 result = Normalizer::isNormalized(field[1], UNORM_NFC, options, status);
361 }
362 if(!result) {
363 errln("Normalizer error: isNormalized(NFC(s), UNORM_NFC) is FALSE");
364 pass = FALSE;
365 }
366 if(field[0]!=field[1] && Normalizer::isNormalized(field[0], UNORM_NFC, options, status)) {
367 errln("Normalizer error: isNormalized(s, UNORM_NFC) is TRUE");
368 pass = FALSE;
369 }
370 if(!Normalizer::isNormalized(field[3], UNORM_NFKC, options, status)) {
371 errln("Normalizer error: isNormalized(NFKC(s), UNORM_NFKC) is FALSE");
372 pass = FALSE;
373 }
374 if(field[0]!=field[3] && Normalizer::isNormalized(field[0], UNORM_NFKC, options, status)) {
375 errln("Normalizer error: isNormalized(s, UNORM_NFKC) is TRUE");
376 pass = FALSE;
377 }
378
379 // test FCD quick check and "makeFCD"
380 Normalizer::normalize(field[0], UNORM_FCD, options, fcd, status);
381 if(UNORM_NO == Normalizer::quickCheck(fcd, UNORM_FCD, options, status)) {
382 errln("Normalizer error: quickCheck(FCD(s), UNORM_FCD) is UNORM_NO");
383 pass = FALSE;
384 }
385 if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_FCD, options, status)) {
386 errln("Normalizer error: quickCheck(NFD(s), UNORM_FCD) is UNORM_NO");
387 pass = FALSE;
388 }
389 if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_FCD, options, status)) {
390 errln("Normalizer error: quickCheck(NFKD(s), UNORM_FCD) is UNORM_NO");
391 pass = FALSE;
392 }
393
394 Normalizer::normalize(fcd, UNORM_NFD, options, out, status);
395 if(out != field[2]) {
396 errln("Normalizer error: NFD(FCD(s))!=NFD(s)");
397 pass = FALSE;
398 }
399
400 if (U_FAILURE(status)) {
401 errln("Normalizer::normalize returned error status");
402 pass = FALSE;
403 }
404
405 if(field[0]!=field[2]) {
406 // two strings that are canonically equivalent must test
407 // equal under a canonical caseless match
408 // see UAX #21 Case Mappings and Jitterbug 2021 and
409 // Unicode Technical Committee meeting consensus 92-C31
410 int32_t rc;
411
412 status=U_ZERO_ERROR;
413 rc=Normalizer::compare(field[0], field[2], (options<<UNORM_COMPARE_NORM_OPTIONS_SHIFT)|U_COMPARE_IGNORE_CASE, status);
414 if(U_FAILURE(status)) {
415 errln("Normalizer::compare(case-insensitive) sets %s", u_errorName(status));
416 pass=FALSE;
417 } else if(rc!=0) {
418 errln("Normalizer::compare(original, NFD, case-insensitive) returned %d instead of 0 for equal", rc);
419 pass=FALSE;
420 }
421 }
422
423 if (!pass) {
424 errln("FAIL: %s", line);
425 }
426 return pass;
427 }
428
429 /**
430 * Do a normalization using the iterative API in the given direction.
431 * @param dir either +1 or -1
432 */
iterativeNorm(const UnicodeString & str,UNormalizationMode mode,int32_t options,UnicodeString & result,int8_t dir)433 void NormalizerConformanceTest::iterativeNorm(const UnicodeString& str,
434 UNormalizationMode mode, int32_t options,
435 UnicodeString& result,
436 int8_t dir) {
437 UErrorCode status = U_ZERO_ERROR;
438 normalizer.setText(str, status);
439 normalizer.setMode(mode);
440 normalizer.setOption(-1, 0); // reset all options
441 normalizer.setOption(options, 1); // set desired options
442 result.truncate(0);
443 if (U_FAILURE(status)) {
444 return;
445 }
446 UChar32 ch;
447 if (dir > 0) {
448 for (ch = normalizer.first(); ch != Normalizer::DONE;
449 ch = normalizer.next()) {
450 result.append(ch);
451 }
452 } else {
453 for (ch = normalizer.last(); ch != Normalizer::DONE;
454 ch = normalizer.previous()) {
455 result.insert(0, ch);
456 }
457 }
458 }
459
460 /**
461 * @param op name of normalization form, e.g., "KC"
462 * @param s string being normalized
463 * @param got value received
464 * @param exp expected value
465 * @param msg description of this test
466 * @param return true if got == exp
467 */
assertEqual(const char * op,const UnicodeString & s,const UnicodeString & got,const UnicodeString & exp,const char * msg,int32_t field)468 UBool NormalizerConformanceTest::assertEqual(const char *op,
469 const UnicodeString& s,
470 const UnicodeString& got,
471 const UnicodeString& exp,
472 const char *msg,
473 int32_t field)
474 {
475 if (exp == got)
476 return TRUE;
477
478 char *sChars, *gotChars, *expChars;
479 UnicodeString sPretty(prettify(s));
480 UnicodeString gotPretty(prettify(got));
481 UnicodeString expPretty(prettify(exp));
482
483 sChars = new char[sPretty.length() + 1];
484 gotChars = new char[gotPretty.length() + 1];
485 expChars = new char[expPretty.length() + 1];
486
487 sPretty.extract(0, sPretty.length(), sChars, sPretty.length() + 1);
488 sChars[sPretty.length()] = 0;
489 gotPretty.extract(0, gotPretty.length(), gotChars, gotPretty.length() + 1);
490 gotChars[gotPretty.length()] = 0;
491 expPretty.extract(0, expPretty.length(), expChars, expPretty.length() + 1);
492 expChars[expPretty.length()] = 0;
493
494 errln(" %s%d)%s(%s)=%s, exp. %s", msg, field, op, sChars, gotChars, expChars);
495
496 delete []sChars;
497 delete []gotChars;
498 delete []expChars;
499 return FALSE;
500 }
501
502 /**
503 * Split a string into pieces based on the given delimiter
504 * character. Then, parse the resultant fields from hex into
505 * characters. That is, "0040 0400;0C00;0899" -> new String[] {
506 * "\u0040\u0400", "\u0C00", "\u0899" }. The output is assumed to
507 * be of the proper length already, and exactly output.length
508 * fields are parsed. If there are too few an exception is
509 * thrown. If there are too many the extras are ignored.
510 *
511 * @return FALSE upon failure
512 */
hexsplit(const char * s,char delimiter,UnicodeString output[],int32_t outputLength)513 UBool NormalizerConformanceTest::hexsplit(const char *s, char delimiter,
514 UnicodeString output[], int32_t outputLength) {
515 const char *t = s;
516 char *end = NULL;
517 UChar32 c;
518 int32_t i;
519 for (i=0; i<outputLength; ++i) {
520 // skip whitespace
521 while(*t == ' ' || *t == '\t') {
522 ++t;
523 }
524
525 // read a sequence of code points
526 output[i].remove();
527 for(;;) {
528 c = (UChar32)uprv_strtoul(t, &end, 16);
529
530 if( (char *)t == end ||
531 (uint32_t)c > 0x10ffff ||
532 (*end != ' ' && *end != '\t' && *end != delimiter)
533 ) {
534 errln(UnicodeString("Bad field ", "") + (i + 1) + " in " + UnicodeString(s, ""));
535 return FALSE;
536 }
537
538 output[i].append(c);
539
540 t = (const char *)end;
541
542 // skip whitespace
543 while(*t == ' ' || *t == '\t') {
544 ++t;
545 }
546
547 if(*t == delimiter) {
548 ++t;
549 break;
550 }
551 if(*t == 0) {
552 if((i + 1) == outputLength) {
553 return TRUE;
554 } else {
555 errln(UnicodeString("Missing field(s) in ", "") + s + " only " + (i + 1) + " out of " + outputLength);
556 return FALSE;
557 }
558 }
559 }
560 }
561 return TRUE;
562 }
563
564 // Specific tests for debugging. These are generally failures taken from
565 // the conformance file, but culled out to make debugging easier.
566
TestCase6(void)567 void NormalizerConformanceTest::TestCase6(void) {
568 _testOneLine("0385;0385;00A8 0301;0020 0308 0301;0020 0308 0301;");
569 }
570
_testOneLine(const char * line)571 void NormalizerConformanceTest::_testOneLine(const char *line) {
572 UErrorCode status = U_ZERO_ERROR;
573 UnicodeString fields[FIELD_COUNT];
574 if (!hexsplit(line, ';', fields, FIELD_COUNT)) {
575 errln((UnicodeString)"Unable to parse line " + line);
576 } else {
577 checkConformance(fields, line, 0, status);
578 }
579 }
580
581 #endif /* #if !UCONFIG_NO_NORMALIZATION */
582