1 /********************************************************************
2 * COPYRIGHT:
3 * Copyright (c) 1997-2011, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
6
7 #include "unicode/ustring.h"
8 #include "unicode/uchar.h"
9 #include "unicode/uniset.h"
10 #include "unicode/putil.h"
11 #include "cstring.h"
12 #include "hash.h"
13 #include "patternprops.h"
14 #include "normalizer2impl.h"
15 #include "uparse.h"
16 #include "ucdtest.h"
17
18 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0]))
19
20 static const char *ignorePropNames[]={
21 "FC_NFKC",
22 "NFD_QC",
23 "NFC_QC",
24 "NFKD_QC",
25 "NFKC_QC",
26 "Expands_On_NFD",
27 "Expands_On_NFC",
28 "Expands_On_NFKD",
29 "Expands_On_NFKC",
30 "NFKC_CF"
31 };
32
UnicodeTest()33 UnicodeTest::UnicodeTest()
34 {
35 UErrorCode errorCode=U_ZERO_ERROR;
36 unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
37 if(U_FAILURE(errorCode)) {
38 delete unknownPropertyNames;
39 unknownPropertyNames=NULL;
40 }
41 // Ignore some property names altogether.
42 for(int32_t i=0; i<LENGTHOF(ignorePropNames); ++i) {
43 unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
44 }
45 }
46
~UnicodeTest()47 UnicodeTest::~UnicodeTest()
48 {
49 delete unknownPropertyNames;
50 }
51
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)52 void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
53 {
54 if(exec) {
55 logln("TestSuite UnicodeTest: ");
56 }
57 TESTCASE_AUTO_BEGIN;
58 TESTCASE_AUTO(TestAdditionalProperties);
59 TESTCASE_AUTO(TestBinaryValues);
60 TESTCASE_AUTO(TestConsistency);
61 TESTCASE_AUTO(TestPatternProperties);
62 TESTCASE_AUTO_END;
63 }
64
65 //====================================================
66 // private data used by the tests
67 //====================================================
68
69 // test DerivedCoreProperties.txt -------------------------------------------
70
71 // copied from genprops.c
72 static int32_t
getTokenIndex(const char * const tokens[],int32_t countTokens,const char * s)73 getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
74 const char *t, *z;
75 int32_t i, j;
76
77 s=u_skipWhitespace(s);
78 for(i=0; i<countTokens; ++i) {
79 t=tokens[i];
80 if(t!=NULL) {
81 for(j=0;; ++j) {
82 if(t[j]!=0) {
83 if(s[j]!=t[j]) {
84 break;
85 }
86 } else {
87 z=u_skipWhitespace(s+j);
88 if(*z==';' || *z==0) {
89 return i;
90 } else {
91 break;
92 }
93 }
94 }
95 }
96 }
97 return -1;
98 }
99
100 static const char *const
101 derivedPropsNames[]={
102 "Math",
103 "Alphabetic",
104 "Lowercase",
105 "Uppercase",
106 "ID_Start",
107 "ID_Continue",
108 "XID_Start",
109 "XID_Continue",
110 "Default_Ignorable_Code_Point",
111 "Full_Composition_Exclusion",
112 "Grapheme_Extend",
113 "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
114 "Grapheme_Base",
115 "Cased",
116 "Case_Ignorable",
117 "Changes_When_Lowercased",
118 "Changes_When_Uppercased",
119 "Changes_When_Titlecased",
120 "Changes_When_Casefolded",
121 "Changes_When_Casemapped",
122 "Changes_When_NFKC_Casefolded"
123 };
124
125 static const UProperty
126 derivedPropsIndex[]={
127 UCHAR_MATH,
128 UCHAR_ALPHABETIC,
129 UCHAR_LOWERCASE,
130 UCHAR_UPPERCASE,
131 UCHAR_ID_START,
132 UCHAR_ID_CONTINUE,
133 UCHAR_XID_START,
134 UCHAR_XID_CONTINUE,
135 UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
136 UCHAR_FULL_COMPOSITION_EXCLUSION,
137 UCHAR_GRAPHEME_EXTEND,
138 UCHAR_GRAPHEME_LINK,
139 UCHAR_GRAPHEME_BASE,
140 UCHAR_CASED,
141 UCHAR_CASE_IGNORABLE,
142 UCHAR_CHANGES_WHEN_LOWERCASED,
143 UCHAR_CHANGES_WHEN_UPPERCASED,
144 UCHAR_CHANGES_WHEN_TITLECASED,
145 UCHAR_CHANGES_WHEN_CASEFOLDED,
146 UCHAR_CHANGES_WHEN_CASEMAPPED,
147 UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
148 };
149
150 static int32_t numErrors[LENGTHOF(derivedPropsIndex)]={ 0 };
151
152 enum { MAX_ERRORS=50 };
153
154 U_CFUNC void U_CALLCONV
derivedPropsLineFn(void * context,char * fields[][2],int32_t,UErrorCode * pErrorCode)155 derivedPropsLineFn(void *context,
156 char *fields[][2], int32_t /* fieldCount */,
157 UErrorCode *pErrorCode)
158 {
159 UnicodeTest *me=(UnicodeTest *)context;
160 uint32_t start, end;
161 int32_t i;
162
163 u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
164 if(U_FAILURE(*pErrorCode)) {
165 me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
166 return;
167 }
168
169 /* parse derived binary property name, ignore unknown names */
170 i=getTokenIndex(derivedPropsNames, LENGTHOF(derivedPropsNames), fields[1][0]);
171 if(i<0) {
172 UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
173 propName.trim();
174 if(me->unknownPropertyNames->find(propName)==NULL) {
175 UErrorCode errorCode=U_ZERO_ERROR;
176 me->unknownPropertyNames->puti(propName, 1, errorCode);
177 me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
178 }
179 return;
180 }
181
182 me->derivedProps[i].add(start, end);
183 }
184
TestAdditionalProperties()185 void UnicodeTest::TestAdditionalProperties() {
186 #if !UCONFIG_NO_NORMALIZATION
187 // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
188 if(LENGTHOF(derivedProps)<LENGTHOF(derivedPropsNames)) {
189 errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
190 LENGTHOF(derivedPropsNames));
191 return;
192 }
193 if(LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)) {
194 errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n");
195 return;
196 }
197
198 char newPath[256];
199 char backupPath[256];
200 char *fields[2][2];
201 UErrorCode errorCode=U_ZERO_ERROR;
202
203 /* Look inside ICU_DATA first */
204 strcpy(newPath, pathToDataDirectory());
205 strcat(newPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
206
207 // As a fallback, try to guess where the source data was located
208 // at the time ICU was built, and look there.
209 # ifdef U_TOPSRCDIR
210 strcpy(backupPath, U_TOPSRCDIR U_FILE_SEP_STRING "data");
211 # else
212 strcpy(backupPath, loadTestData(errorCode));
213 strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data");
214 # endif
215 strcat(backupPath, U_FILE_SEP_STRING);
216 strcat(backupPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
217
218 char *path=newPath;
219 u_parseDelimitedFile(newPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
220
221 if(errorCode==U_FILE_ACCESS_ERROR) {
222 errorCode=U_ZERO_ERROR;
223 path=backupPath;
224 u_parseDelimitedFile(backupPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
225 }
226 if(U_FAILURE(errorCode)) {
227 errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
228 return;
229 }
230 char *basename=path+strlen(path)-strlen("DerivedCoreProperties.txt");
231 strcpy(basename, "DerivedNormalizationProps.txt");
232 u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
233 if(U_FAILURE(errorCode)) {
234 errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
235 return;
236 }
237
238 // now we have all derived core properties in the UnicodeSets
239 // run them all through the API
240 int32_t rangeCount, range;
241 uint32_t i;
242 UChar32 start, end;
243
244 // test all TRUE properties
245 for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
246 rangeCount=derivedProps[i].getRangeCount();
247 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
248 start=derivedProps[i].getRangeStart(range);
249 end=derivedProps[i].getRangeEnd(range);
250 for(; start<=end; ++start) {
251 if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
252 dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
253 if(++numErrors[i]>=MAX_ERRORS) {
254 dataerrln("Too many errors, moving to the next test");
255 break;
256 }
257 }
258 }
259 }
260 }
261
262 // invert all properties
263 for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
264 derivedProps[i].complement();
265 }
266
267 // test all FALSE properties
268 for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
269 rangeCount=derivedProps[i].getRangeCount();
270 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
271 start=derivedProps[i].getRangeStart(range);
272 end=derivedProps[i].getRangeEnd(range);
273 for(; start<=end; ++start) {
274 if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
275 errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
276 if(++numErrors[i]>=MAX_ERRORS) {
277 errln("Too many errors, moving to the next test");
278 break;
279 }
280 }
281 }
282 }
283 }
284 #endif /* !UCONFIG_NO_NORMALIZATION */
285 }
286
TestBinaryValues()287 void UnicodeTest::TestBinaryValues() {
288 /*
289 * Unicode 5.1 explicitly defines binary property value aliases.
290 * Verify that they are all recognized.
291 */
292 UErrorCode errorCode=U_ZERO_ERROR;
293 UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
294 if(U_FAILURE(errorCode)) {
295 dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
296 return;
297 }
298
299 static const char *const falseValues[]={ "N", "No", "F", "False" };
300 static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
301 int32_t i;
302 for(i=0; i<LENGTHOF(falseValues); ++i) {
303 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
304 pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
305 errorCode=U_ZERO_ERROR;
306 UnicodeSet set(pattern, errorCode);
307 if(U_FAILURE(errorCode)) {
308 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
309 continue;
310 }
311 set.complement();
312 if(set!=alpha) {
313 errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
314 }
315 }
316 for(i=0; i<LENGTHOF(trueValues); ++i) {
317 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
318 pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
319 errorCode=U_ZERO_ERROR;
320 UnicodeSet set(pattern, errorCode);
321 if(U_FAILURE(errorCode)) {
322 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
323 continue;
324 }
325 if(set!=alpha) {
326 errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
327 }
328 }
329 }
330
TestConsistency()331 void UnicodeTest::TestConsistency() {
332 #if !UCONFIG_NO_NORMALIZATION
333 /*
334 * Test for an example that getCanonStartSet() delivers
335 * all characters that compose from the input one,
336 * even in multiple steps.
337 * For example, the set for "I" (0049) should contain both
338 * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
339 * In general, the set for the middle such character should be a subset
340 * of the set for the first.
341 */
342 IcuTestErrorCode errorCode(*this, "TestConsistency");
343 const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
344 const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
345 if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
346 dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
347 errorCode.errorName());
348 errorCode.reset();
349 return;
350 }
351
352 UnicodeSet set1, set2;
353 if (nfcImpl->getCanonStartSet(0x49, set1)) {
354 /* enumerate all characters that are plausible to be latin letters */
355 for(UChar start=0xa0; start<0x2000; ++start) {
356 UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
357 if(decomp.length()>1 && decomp[0]==0x49) {
358 set2.add(start);
359 }
360 }
361
362 if (set1!=set2) {
363 errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
364 }
365 // This was available in cucdtst.c but the test had to move to intltest
366 // because the new internal normalization functions are in C++.
367 //compareUSets(set1, set2,
368 // "[canon start set of 0049]", "[all c with canon decomp with 0049]",
369 // TRUE);
370 } else {
371 errln("NFC.getCanonStartSet() returned FALSE");
372 }
373 #endif
374 }
375
376 /**
377 * Test various implementations of Pattern_Syntax & Pattern_White_Space.
378 */
TestPatternProperties()379 void UnicodeTest::TestPatternProperties() {
380 IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
381 UnicodeSet syn_pp;
382 UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
383 UnicodeSet syn_list(
384 "[!-/\\:-@\\[-\\^`\\{-~"
385 "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
386 "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
387 "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
388 UnicodeSet ws_pp;
389 UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
390 UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
391 UnicodeSet syn_ws_pp;
392 UnicodeSet syn_ws_prop(syn_prop);
393 syn_ws_prop.addAll(ws_prop);
394 for(UChar32 c=0; c<=0xffff; ++c) {
395 if(PatternProps::isSyntax(c)) {
396 syn_pp.add(c);
397 }
398 if(PatternProps::isWhiteSpace(c)) {
399 ws_pp.add(c);
400 }
401 if(PatternProps::isSyntaxOrWhiteSpace(c)) {
402 syn_ws_pp.add(c);
403 }
404 }
405 compareUSets(syn_pp, syn_prop,
406 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
407 compareUSets(syn_pp, syn_list,
408 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
409 compareUSets(ws_pp, ws_prop,
410 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
411 compareUSets(ws_pp, ws_list,
412 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
413 compareUSets(syn_ws_pp, syn_ws_prop,
414 "PatternProps.isSyntaxOrWhiteSpace()",
415 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
416 }
417
418 // So far only minimal port of Java & cucdtst.c compareUSets().
419 UBool
compareUSets(const UnicodeSet & a,const UnicodeSet & b,const char * a_name,const char * b_name,UBool diffIsError)420 UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
421 const char *a_name, const char *b_name,
422 UBool diffIsError) {
423 UBool same= a==b;
424 if(!same && diffIsError) {
425 errln("Sets are different: %s vs. %s\n", a_name, b_name);
426 }
427 return same;
428 }
429