1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /********************************************************************
4 * Copyright (c) 1997-2016, International Business Machines Corporation and
5 * others. All Rights Reserved.
6 ********************************************************************/
7
8 #include "unicode/ustring.h"
9 #include "unicode/uchar.h"
10 #include "unicode/ucpmap.h"
11 #include "unicode/uniset.h"
12 #include "unicode/putil.h"
13 #include "unicode/uscript.h"
14 #include "unicode/uset.h"
15 #include "cstring.h"
16 #include "hash.h"
17 #include "patternprops.h"
18 #include "normalizer2impl.h"
19 #include "testutil.h"
20 #include "uparse.h"
21 #include "ucdtest.h"
22
23 static const char *ignorePropNames[]={
24 "FC_NFKC",
25 "NFD_QC",
26 "NFC_QC",
27 "NFKD_QC",
28 "NFKC_QC",
29 "Expands_On_NFD",
30 "Expands_On_NFC",
31 "Expands_On_NFKD",
32 "Expands_On_NFKC",
33 "NFKC_CF"
34 };
35
UnicodeTest()36 UnicodeTest::UnicodeTest()
37 {
38 UErrorCode errorCode=U_ZERO_ERROR;
39 unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
40 if(U_FAILURE(errorCode)) {
41 delete unknownPropertyNames;
42 unknownPropertyNames=NULL;
43 }
44 // Ignore some property names altogether.
45 for(int32_t i=0; i<UPRV_LENGTHOF(ignorePropNames); ++i) {
46 unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
47 }
48 }
49
~UnicodeTest()50 UnicodeTest::~UnicodeTest()
51 {
52 delete unknownPropertyNames;
53 }
54
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)55 void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
56 {
57 if(exec) {
58 logln("TestSuite UnicodeTest: ");
59 }
60 TESTCASE_AUTO_BEGIN;
61 TESTCASE_AUTO(TestAdditionalProperties);
62 TESTCASE_AUTO(TestBinaryValues);
63 TESTCASE_AUTO(TestConsistency);
64 TESTCASE_AUTO(TestPatternProperties);
65 TESTCASE_AUTO(TestScriptMetadata);
66 TESTCASE_AUTO(TestBidiPairedBracketType);
67 TESTCASE_AUTO(TestEmojiProperties);
68 TESTCASE_AUTO(TestIndicPositionalCategory);
69 TESTCASE_AUTO(TestIndicSyllabicCategory);
70 TESTCASE_AUTO(TestVerticalOrientation);
71 TESTCASE_AUTO(TestDefaultScriptExtensions);
72 TESTCASE_AUTO(TestInvalidCodePointFolding);
73 #if !UCONFIG_NO_NORMALIZATION
74 TESTCASE_AUTO(TestBinaryCharacterProperties);
75 TESTCASE_AUTO(TestIntCharacterProperties);
76 #endif
77 TESTCASE_AUTO_END;
78 }
79
80 //====================================================
81 // private data used by the tests
82 //====================================================
83
84 // test DerivedCoreProperties.txt -------------------------------------------
85
86 // copied from genprops.c
87 static int32_t
getTokenIndex(const char * const tokens[],int32_t countTokens,const char * s)88 getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
89 const char *t, *z;
90 int32_t i, j;
91
92 s=u_skipWhitespace(s);
93 for(i=0; i<countTokens; ++i) {
94 t=tokens[i];
95 if(t!=NULL) {
96 for(j=0;; ++j) {
97 if(t[j]!=0) {
98 if(s[j]!=t[j]) {
99 break;
100 }
101 } else {
102 z=u_skipWhitespace(s+j);
103 if(*z==';' || *z==0) {
104 return i;
105 } else {
106 break;
107 }
108 }
109 }
110 }
111 }
112 return -1;
113 }
114
115 static const char *const
116 derivedPropsNames[]={
117 "Math",
118 "Alphabetic",
119 "Lowercase",
120 "Uppercase",
121 "ID_Start",
122 "ID_Continue",
123 "XID_Start",
124 "XID_Continue",
125 "Default_Ignorable_Code_Point",
126 "Full_Composition_Exclusion",
127 "Grapheme_Extend",
128 "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
129 "Grapheme_Base",
130 "Cased",
131 "Case_Ignorable",
132 "Changes_When_Lowercased",
133 "Changes_When_Uppercased",
134 "Changes_When_Titlecased",
135 "Changes_When_Casefolded",
136 "Changes_When_Casemapped",
137 "Changes_When_NFKC_Casefolded"
138 };
139
140 static const UProperty
141 derivedPropsIndex[]={
142 UCHAR_MATH,
143 UCHAR_ALPHABETIC,
144 UCHAR_LOWERCASE,
145 UCHAR_UPPERCASE,
146 UCHAR_ID_START,
147 UCHAR_ID_CONTINUE,
148 UCHAR_XID_START,
149 UCHAR_XID_CONTINUE,
150 UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
151 UCHAR_FULL_COMPOSITION_EXCLUSION,
152 UCHAR_GRAPHEME_EXTEND,
153 UCHAR_GRAPHEME_LINK,
154 UCHAR_GRAPHEME_BASE,
155 UCHAR_CASED,
156 UCHAR_CASE_IGNORABLE,
157 UCHAR_CHANGES_WHEN_LOWERCASED,
158 UCHAR_CHANGES_WHEN_UPPERCASED,
159 UCHAR_CHANGES_WHEN_TITLECASED,
160 UCHAR_CHANGES_WHEN_CASEFOLDED,
161 UCHAR_CHANGES_WHEN_CASEMAPPED,
162 UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
163 };
164
165 static int32_t numErrors[UPRV_LENGTHOF(derivedPropsIndex)]={ 0 };
166
167 enum { MAX_ERRORS=50 };
168
169 U_CFUNC void U_CALLCONV
derivedPropsLineFn(void * context,char * fields[][2],int32_t,UErrorCode * pErrorCode)170 derivedPropsLineFn(void *context,
171 char *fields[][2], int32_t /* fieldCount */,
172 UErrorCode *pErrorCode)
173 {
174 UnicodeTest *me=(UnicodeTest *)context;
175 uint32_t start, end;
176 int32_t i;
177
178 u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
179 if(U_FAILURE(*pErrorCode)) {
180 me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
181 return;
182 }
183
184 /* parse derived binary property name, ignore unknown names */
185 i=getTokenIndex(derivedPropsNames, UPRV_LENGTHOF(derivedPropsNames), fields[1][0]);
186 if(i<0) {
187 UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
188 propName.trim();
189 if(me->unknownPropertyNames->find(propName)==NULL) {
190 UErrorCode errorCode=U_ZERO_ERROR;
191 me->unknownPropertyNames->puti(propName, 1, errorCode);
192 me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
193 }
194 return;
195 }
196
197 me->derivedProps[i].add(start, end);
198 }
199
TestAdditionalProperties()200 void UnicodeTest::TestAdditionalProperties() {
201 #if !UCONFIG_NO_NORMALIZATION
202 // Android patch: b/145129186 Disable failing tests
203 #ifndef ANDROID
204 // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
205 if(UPRV_LENGTHOF(derivedProps)<UPRV_LENGTHOF(derivedPropsNames)) {
206 errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
207 UPRV_LENGTHOF(derivedPropsNames));
208 return;
209 }
210 if(UPRV_LENGTHOF(derivedPropsIndex)!=UPRV_LENGTHOF(derivedPropsNames)) {
211 errln("error in ucdtest.cpp: UPRV_LENGTHOF(derivedPropsIndex)!=UPRV_LENGTHOF(derivedPropsNames)\n");
212 return;
213 }
214
215 char path[500];
216 if(getUnidataPath(path) == NULL) {
217 errln("unable to find path to source/data/unidata/");
218 return;
219 }
220 char *basename=strchr(path, 0);
221 strcpy(basename, "DerivedCoreProperties.txt");
222
223 char *fields[2][2];
224 UErrorCode errorCode=U_ZERO_ERROR;
225 u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
226 if(U_FAILURE(errorCode)) {
227 errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
228 return;
229 }
230
231 strcpy(basename, "DerivedNormalizationProps.txt");
232 u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
233 if(U_FAILURE(errorCode)) {
234 errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
235 return;
236 }
237
238 // now we have all derived core properties in the UnicodeSets
239 // run them all through the API
240 int32_t rangeCount, range;
241 uint32_t i;
242 UChar32 start, end;
243
244 // test all TRUE properties
245 for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
246 rangeCount=derivedProps[i].getRangeCount();
247 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
248 start=derivedProps[i].getRangeStart(range);
249 end=derivedProps[i].getRangeEnd(range);
250 for(; start<=end; ++start) {
251 if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
252 dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
253 if(++numErrors[i]>=MAX_ERRORS) {
254 dataerrln("Too many errors, moving to the next test");
255 break;
256 }
257 }
258 }
259 }
260 }
261
262 // invert all properties
263 for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
264 derivedProps[i].complement();
265 }
266
267 // test all FALSE properties
268 for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
269 rangeCount=derivedProps[i].getRangeCount();
270 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
271 start=derivedProps[i].getRangeStart(range);
272 end=derivedProps[i].getRangeEnd(range);
273 for(; start<=end; ++start) {
274 if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
275 errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
276 if(++numErrors[i]>=MAX_ERRORS) {
277 errln("Too many errors, moving to the next test");
278 break;
279 }
280 }
281 }
282 }
283 }
284 #endif /* ANDROID */
285 #endif /* !UCONFIG_NO_NORMALIZATION */
286 }
287
TestBinaryValues()288 void UnicodeTest::TestBinaryValues() {
289 /*
290 * Unicode 5.1 explicitly defines binary property value aliases.
291 * Verify that they are all recognized.
292 */
293 UErrorCode errorCode=U_ZERO_ERROR;
294 UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
295 if(U_FAILURE(errorCode)) {
296 dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
297 return;
298 }
299
300 static const char *const falseValues[]={ "N", "No", "F", "False" };
301 static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
302 int32_t i;
303 for(i=0; i<UPRV_LENGTHOF(falseValues); ++i) {
304 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
305 pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
306 errorCode=U_ZERO_ERROR;
307 UnicodeSet set(pattern, errorCode);
308 if(U_FAILURE(errorCode)) {
309 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
310 continue;
311 }
312 set.complement();
313 if(set!=alpha) {
314 errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
315 }
316 }
317 for(i=0; i<UPRV_LENGTHOF(trueValues); ++i) {
318 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
319 pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
320 errorCode=U_ZERO_ERROR;
321 UnicodeSet set(pattern, errorCode);
322 if(U_FAILURE(errorCode)) {
323 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
324 continue;
325 }
326 if(set!=alpha) {
327 errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
328 }
329 }
330 }
331
TestConsistency()332 void UnicodeTest::TestConsistency() {
333 #if !UCONFIG_NO_NORMALIZATION
334 /*
335 * Test for an example that getCanonStartSet() delivers
336 * all characters that compose from the input one,
337 * even in multiple steps.
338 * For example, the set for "I" (0049) should contain both
339 * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
340 * In general, the set for the middle such character should be a subset
341 * of the set for the first.
342 */
343 IcuTestErrorCode errorCode(*this, "TestConsistency");
344 const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
345 const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
346 if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
347 dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
348 errorCode.errorName());
349 errorCode.reset();
350 return;
351 }
352
353 UnicodeSet set1, set2;
354 if (nfcImpl->getCanonStartSet(0x49, set1)) {
355 /* enumerate all characters that are plausible to be latin letters */
356 for(UChar start=0xa0; start<0x2000; ++start) {
357 UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
358 if(decomp.length()>1 && decomp[0]==0x49) {
359 set2.add(start);
360 }
361 }
362
363 if (set1!=set2) {
364 errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
365 }
366 // This was available in cucdtst.c but the test had to move to intltest
367 // because the new internal normalization functions are in C++.
368 //compareUSets(set1, set2,
369 // "[canon start set of 0049]", "[all c with canon decomp with 0049]",
370 // TRUE);
371 } else {
372 errln("NFC.getCanonStartSet() returned FALSE");
373 }
374 #endif
375 }
376
377 /**
378 * Test various implementations of Pattern_Syntax & Pattern_White_Space.
379 */
TestPatternProperties()380 void UnicodeTest::TestPatternProperties() {
381 IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
382 UnicodeSet syn_pp;
383 UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
384 UnicodeSet syn_list(
385 "[!-/\\:-@\\[-\\^`\\{-~"
386 "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
387 "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
388 "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
389 UnicodeSet ws_pp;
390 UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
391 UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
392 UnicodeSet syn_ws_pp;
393 UnicodeSet syn_ws_prop(syn_prop);
394 syn_ws_prop.addAll(ws_prop);
395 for(UChar32 c=0; c<=0xffff; ++c) {
396 if(PatternProps::isSyntax(c)) {
397 syn_pp.add(c);
398 }
399 if(PatternProps::isWhiteSpace(c)) {
400 ws_pp.add(c);
401 }
402 if(PatternProps::isSyntaxOrWhiteSpace(c)) {
403 syn_ws_pp.add(c);
404 }
405 }
406 compareUSets(syn_pp, syn_prop,
407 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
408 compareUSets(syn_pp, syn_list,
409 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
410 compareUSets(ws_pp, ws_prop,
411 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
412 compareUSets(ws_pp, ws_list,
413 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
414 compareUSets(syn_ws_pp, syn_ws_prop,
415 "PatternProps.isSyntaxOrWhiteSpace()",
416 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
417 }
418
419 // So far only minimal port of Java & cucdtst.c compareUSets().
420 UBool
compareUSets(const UnicodeSet & a,const UnicodeSet & b,const char * a_name,const char * b_name,UBool diffIsError)421 UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
422 const char *a_name, const char *b_name,
423 UBool diffIsError) {
424 UBool same= a==b;
425 if(!same && diffIsError) {
426 errln("Sets are different: %s vs. %s\n", a_name, b_name);
427 }
428 return same;
429 }
430
431 namespace {
432
433 /**
434 * Maps a special script code to the most common script of its encoded characters.
435 */
getCharScript(UScriptCode script)436 UScriptCode getCharScript(UScriptCode script) {
437 switch(script) {
438 case USCRIPT_HAN_WITH_BOPOMOFO:
439 case USCRIPT_SIMPLIFIED_HAN:
440 case USCRIPT_TRADITIONAL_HAN:
441 return USCRIPT_HAN;
442 case USCRIPT_JAPANESE:
443 return USCRIPT_HIRAGANA;
444 case USCRIPT_JAMO:
445 case USCRIPT_KOREAN:
446 return USCRIPT_HANGUL;
447 case USCRIPT_SYMBOLS_EMOJI:
448 return USCRIPT_SYMBOLS;
449 default:
450 return script;
451 }
452 }
453
454 } // namespace
455
TestScriptMetadata()456 void UnicodeTest::TestScriptMetadata() {
457 IcuTestErrorCode errorCode(*this, "TestScriptMetadata()");
458 UnicodeSet rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode);
459 // So far, sample characters are uppercase.
460 // Georgian is special.
461 UnicodeSet cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode);
462 for(int32_t sci = 0; sci < USCRIPT_CODE_LIMIT; ++sci) {
463 UScriptCode sc = (UScriptCode)sci;
464 // Run the test with -v to see which script has failures:
465 // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 6 FAIL
466 logln(uscript_getShortName(sc));
467 UScriptUsage usage = uscript_getUsage(sc);
468 UnicodeString sample = uscript_getSampleUnicodeString(sc);
469 UnicodeSet scriptSet;
470 scriptSet.applyIntPropertyValue(UCHAR_SCRIPT, sc, errorCode);
471 if(usage == USCRIPT_USAGE_NOT_ENCODED) {
472 assertTrue("not encoded, no sample", sample.isEmpty());
473 assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc));
474 assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc));
475 assertFalse("not encoded, not cased", uscript_isCased(sc));
476 assertTrue("not encoded, no characters", scriptSet.isEmpty());
477 } else {
478 assertFalse("encoded, has a sample character", sample.isEmpty());
479 UChar32 firstChar = sample.char32At(0);
480 UScriptCode charScript = getCharScript(sc);
481 assertEquals("script(sample(script))",
482 (int32_t)charScript, (int32_t)uscript_getScript(firstChar, errorCode));
483 assertEquals("RTL vs. set", (UBool)rtl.contains(firstChar), (UBool)uscript_isRightToLeft(sc));
484 assertEquals("cased vs. set", (UBool)cased.contains(firstChar), (UBool)uscript_isCased(sc));
485 assertEquals("encoded, has characters", (UBool)(sc == charScript), (UBool)(!scriptSet.isEmpty()));
486 if(uscript_isRightToLeft(sc)) {
487 rtl.removeAll(scriptSet);
488 }
489 if(uscript_isCased(sc)) {
490 cased.removeAll(scriptSet);
491 }
492 }
493 }
494 UnicodeString pattern;
495 assertEquals("no remaining RTL characters",
496 UnicodeString("[]"), rtl.toPattern(pattern));
497 assertEquals("no remaining cased characters",
498 UnicodeString("[]"), cased.toPattern(pattern));
499
500 assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN));
501 assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI));
502 assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN));
503 }
504
TestBidiPairedBracketType()505 void UnicodeTest::TestBidiPairedBracketType() {
506 // BidiBrackets-6.3.0.txt says:
507 //
508 // The set of code points listed in this file was originally derived
509 // using the character properties General_Category (gc), Bidi_Class (bc),
510 // Bidi_Mirrored (Bidi_M), and Bidi_Mirroring_Glyph (bmg), as follows:
511 // two characters, A and B, form a pair if A has gc=Ps and B has gc=Pe,
512 // both have bc=ON and Bidi_M=Y, and bmg of A is B. Bidi_Paired_Bracket
513 // maps A to B and vice versa, and their Bidi_Paired_Bracket_Type
514 // property values are Open and Close, respectively.
515 IcuTestErrorCode errorCode(*this, "TestBidiPairedBracketType()");
516 UnicodeSet bpt("[:^bpt=n:]", errorCode);
517 assertTrue("bpt!=None is not empty", !bpt.isEmpty());
518 // The following should always be true.
519 UnicodeSet mirrored("[:Bidi_M:]", errorCode);
520 UnicodeSet other_neutral("[:bc=ON:]", errorCode);
521 assertTrue("bpt!=None is a subset of Bidi_M", mirrored.containsAll(bpt));
522 assertTrue("bpt!=None is a subset of bc=ON", other_neutral.containsAll(bpt));
523 // The following are true at least initially in Unicode 6.3.
524 UnicodeSet bpt_open("[:bpt=o:]", errorCode);
525 UnicodeSet bpt_close("[:bpt=c:]", errorCode);
526 UnicodeSet ps("[:Ps:]", errorCode);
527 UnicodeSet pe("[:Pe:]", errorCode);
528 assertTrue("bpt=Open is a subset of Ps", ps.containsAll(bpt_open));
529 assertTrue("bpt=Close is a subset of Pe", pe.containsAll(bpt_close));
530 }
531
TestEmojiProperties()532 void UnicodeTest::TestEmojiProperties() {
533 assertFalse("space is not Emoji", u_hasBinaryProperty(0x20, UCHAR_EMOJI));
534 assertTrue("shooting star is Emoji", u_hasBinaryProperty(0x1F320, UCHAR_EMOJI));
535 IcuTestErrorCode errorCode(*this, "TestEmojiProperties()");
536 UnicodeSet emoji("[:Emoji:]", errorCode);
537 assertTrue("lots of Emoji", emoji.size() > 700);
538
539 assertTrue("shooting star is Emoji_Presentation",
540 u_hasBinaryProperty(0x1F320, UCHAR_EMOJI_PRESENTATION));
541 assertTrue("Fitzpatrick 6 is Emoji_Modifier",
542 u_hasBinaryProperty(0x1F3FF, UCHAR_EMOJI_MODIFIER));
543 assertTrue("happy person is Emoji_Modifier_Base",
544 u_hasBinaryProperty(0x1F64B, UCHAR_EMOJI_MODIFIER_BASE));
545 assertTrue("asterisk is Emoji_Component",
546 u_hasBinaryProperty(0x2A, UCHAR_EMOJI_COMPONENT));
547 assertTrue("copyright is Extended_Pictographic",
548 u_hasBinaryProperty(0xA9, UCHAR_EXTENDED_PICTOGRAPHIC));
549 }
550
TestIndicPositionalCategory()551 void UnicodeTest::TestIndicPositionalCategory() {
552 IcuTestErrorCode errorCode(*this, "TestIndicPositionalCategory()");
553 UnicodeSet na(u"[:InPC=NA:]", errorCode);
554 assertTrue("mostly NA", 1000000 <= na.size() && na.size() <= UCHAR_MAX_VALUE - 500);
555 UnicodeSet vol(u"[:InPC=Visual_Order_Left:]", errorCode);
556 assertTrue("some Visual_Order_Left", 19 <= vol.size() && vol.size() <= 100);
557 assertEquals("U+08FF: NA", U_INPC_NA,
558 u_getIntPropertyValue(0x08FF, UCHAR_INDIC_POSITIONAL_CATEGORY));
559 assertEquals("U+0900: Top", U_INPC_TOP,
560 u_getIntPropertyValue(0x0900, UCHAR_INDIC_POSITIONAL_CATEGORY));
561 assertEquals("U+10A06: Overstruck", U_INPC_OVERSTRUCK,
562 u_getIntPropertyValue(0x10A06, UCHAR_INDIC_POSITIONAL_CATEGORY));
563 }
564
TestIndicSyllabicCategory()565 void UnicodeTest::TestIndicSyllabicCategory() {
566 IcuTestErrorCode errorCode(*this, "TestIndicSyllabicCategory()");
567 UnicodeSet other(u"[:InSC=Other:]", errorCode);
568 assertTrue("mostly Other", 1000000 <= other.size() && other.size() <= UCHAR_MAX_VALUE - 500);
569 UnicodeSet ava(u"[:InSC=Avagraha:]", errorCode);
570 assertTrue("some Avagraha", 16 <= ava.size() && ava.size() <= 100);
571 assertEquals("U+08FF: Other", U_INSC_OTHER,
572 u_getIntPropertyValue(0x08FF, UCHAR_INDIC_SYLLABIC_CATEGORY));
573 assertEquals("U+0900: Bindu", U_INSC_BINDU,
574 u_getIntPropertyValue(0x0900, UCHAR_INDIC_SYLLABIC_CATEGORY));
575 assertEquals("U+11065: Brahmi_Joining_Number", U_INSC_BRAHMI_JOINING_NUMBER,
576 u_getIntPropertyValue(0x11065, UCHAR_INDIC_SYLLABIC_CATEGORY));
577 }
578
TestVerticalOrientation()579 void UnicodeTest::TestVerticalOrientation() {
580 IcuTestErrorCode errorCode(*this, "TestVerticalOrientation()");
581 UnicodeSet r(u"[:vo=R:]", errorCode);
582 assertTrue("mostly R", 0xc0000 <= r.size() && r.size() <= 0xd0000);
583 UnicodeSet u(u"[:vo=U:]", errorCode);
584 assertTrue("much U", 0x40000 <= u.size() && u.size() <= 0x50000);
585 UnicodeSet tu(u"[:vo=Tu:]", errorCode);
586 assertTrue("some Tu", 147 <= tu.size() && tu.size() <= 300);
587 assertEquals("U+0E01: Rotated", U_VO_ROTATED,
588 u_getIntPropertyValue(0x0E01, UCHAR_VERTICAL_ORIENTATION));
589 assertEquals("U+3008: Transformed_Rotated", U_VO_TRANSFORMED_ROTATED,
590 u_getIntPropertyValue(0x3008, UCHAR_VERTICAL_ORIENTATION));
591 assertEquals("U+33333: Upright", U_VO_UPRIGHT,
592 u_getIntPropertyValue(0x33333, UCHAR_VERTICAL_ORIENTATION));
593 }
594
TestDefaultScriptExtensions()595 void UnicodeTest::TestDefaultScriptExtensions() {
596 // Block 3000..303F CJK Symbols and Punctuation defaults to scx=Bopo Hang Hani Hira Kana Yiii
597 // but some of its characters revert to scx=<script> which is usually Common.
598 IcuTestErrorCode errorCode(*this, "TestDefaultScriptExtensions()");
599 UScriptCode scx[20];
600 scx[0] = USCRIPT_INVALID_CODE;
601 assertEquals("U+3000 num scx", 1, // IDEOGRAPHIC SPACE
602 uscript_getScriptExtensions(0x3000, scx, UPRV_LENGTHOF(scx), errorCode));
603 assertEquals("U+3000 num scx[0]", USCRIPT_COMMON, scx[0]);
604 scx[0] = USCRIPT_INVALID_CODE;
605 assertEquals("U+3012 num scx", 1, // POSTAL MARK
606 uscript_getScriptExtensions(0x3012, scx, UPRV_LENGTHOF(scx), errorCode));
607 assertEquals("U+3012 num scx[0]", USCRIPT_COMMON, scx[0]);
608 }
609
TestInvalidCodePointFolding(void)610 void UnicodeTest::TestInvalidCodePointFolding(void) {
611 // Test behavior when an invalid code point is passed to u_foldCase
612 static const UChar32 invalidCodePoints[] = {
613 0xD800, // lead surrogate
614 0xDFFF, // trail surrogate
615 0xFDD0, // noncharacter
616 0xFFFF, // noncharacter
617 0x110000, // out of range
618 -1 // negative
619 };
620 for (int32_t i=0; i<UPRV_LENGTHOF(invalidCodePoints); ++i) {
621 UChar32 cp = invalidCodePoints[i];
622 assertEquals("Invalid code points should be echoed back",
623 cp, u_foldCase(cp, U_FOLD_CASE_DEFAULT));
624 assertEquals("Invalid code points should be echoed back",
625 cp, u_foldCase(cp, U_FOLD_CASE_EXCLUDE_SPECIAL_I));
626 }
627 }
628
TestBinaryCharacterProperties()629 void UnicodeTest::TestBinaryCharacterProperties() {
630 #if !UCONFIG_NO_NORMALIZATION
631 IcuTestErrorCode errorCode(*this, "TestBinaryCharacterProperties()");
632 // Spot-check getBinaryPropertySet() vs. hasBinaryProperty().
633 for (int32_t prop = 0; prop < UCHAR_BINARY_LIMIT; ++prop) {
634 const USet *uset = u_getBinaryPropertySet((UProperty)prop, errorCode);
635 if (errorCode.errIfFailureAndReset("u_getBinaryPropertySet(%d)", (int)prop)) {
636 continue;
637 }
638 const UnicodeSet &set = *UnicodeSet::fromUSet(uset);
639 int32_t size = set.size();
640 if (size == 0) {
641 assertFalse(UnicodeString("!hasBinaryProperty(U+0020, ") + prop + u")",
642 u_hasBinaryProperty(0x20, (UProperty)prop));
643 assertFalse(UnicodeString("!hasBinaryProperty(U+0061, ") + prop + u")",
644 u_hasBinaryProperty(0x61, (UProperty)prop));
645 assertFalse(UnicodeString("!hasBinaryProperty(U+4E00, ") + prop + u")",
646 u_hasBinaryProperty(0x4e00, (UProperty)prop));
647 } else {
648 UChar32 c = set.charAt(0);
649 if (c > 0) {
650 assertFalse(
651 UnicodeString("!hasBinaryProperty(") + TestUtility::hex(c - 1) +
652 u", " + prop + u")",
653 u_hasBinaryProperty(c - 1, (UProperty)prop));
654 }
655 assertTrue(
656 UnicodeString("hasBinaryProperty(") + TestUtility::hex(c) +
657 u", " + prop + u")",
658 u_hasBinaryProperty(c, (UProperty)prop));
659 c = set.charAt(size - 1);
660 assertTrue(
661 UnicodeString("hasBinaryProperty(") + TestUtility::hex(c) +
662 u", " + prop + u")",
663 u_hasBinaryProperty(c, (UProperty)prop));
664 if (c < 0x10ffff) {
665 assertFalse(
666 UnicodeString("!hasBinaryProperty(") + TestUtility::hex(c + 1) +
667 u", " + prop + u")",
668 u_hasBinaryProperty(c + 1, (UProperty)prop));
669 }
670 }
671 }
672 #endif
673 }
674
TestIntCharacterProperties()675 void UnicodeTest::TestIntCharacterProperties() {
676 #if !UCONFIG_NO_NORMALIZATION
677 IcuTestErrorCode errorCode(*this, "TestIntCharacterProperties()");
678 // Spot-check getIntPropertyMap() vs. getIntPropertyValue().
679 for (int32_t prop = UCHAR_INT_START; prop < UCHAR_INT_LIMIT; ++prop) {
680 const UCPMap *map = u_getIntPropertyMap((UProperty)prop, errorCode);
681 if (errorCode.errIfFailureAndReset("u_getIntPropertyMap(%d)", (int)prop)) {
682 continue;
683 }
684 uint32_t value;
685 UChar32 end = ucpmap_getRange(map, 0, UCPMAP_RANGE_NORMAL, 0, nullptr, nullptr, &value);
686 assertTrue("int property first range", end >= 0);
687 UChar32 c = end / 2;
688 assertEquals(UnicodeString("int property first range value at ") + TestUtility::hex(c),
689 u_getIntPropertyValue(c, (UProperty)prop), value);
690 end = ucpmap_getRange(map, 0x5000, UCPMAP_RANGE_NORMAL, 0, nullptr, nullptr, &value);
691 assertTrue("int property later range", end >= 0);
692 assertEquals(UnicodeString("int property later range value at ") + TestUtility::hex(end),
693 u_getIntPropertyValue(end, (UProperty)prop), value);
694 // ucpmap_get() API coverage
695 // TODO: move to cucdtst.c
696 assertEquals(
697 "int property upcmap_get(U+0061)",
698 u_getIntPropertyValue(0x61, (UProperty)prop), ucpmap_get(map, 0x61));
699 }
700 #endif
701 }
702