1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /********************************************************************
4 * Copyright (c) 1997-2016, International Business Machines Corporation and
5 * others. All Rights Reserved.
6 ********************************************************************/
7
8 #include "unicode/ustring.h"
9 #include "unicode/uchar.h"
10 #include "unicode/ucpmap.h"
11 #include "unicode/uniset.h"
12 #include "unicode/putil.h"
13 #include "unicode/uscript.h"
14 #include "unicode/uset.h"
15 #include "cstring.h"
16 #include "hash.h"
17 #include "patternprops.h"
18 #include "normalizer2impl.h"
19 #include "testutil.h"
20 #include "uparse.h"
21 #include "ucdtest.h"
22
23 static const char *ignorePropNames[]={
24 "FC_NFKC",
25 "NFD_QC",
26 "NFC_QC",
27 "NFKD_QC",
28 "NFKC_QC",
29 "Expands_On_NFD",
30 "Expands_On_NFC",
31 "Expands_On_NFKD",
32 "Expands_On_NFKC",
33 "NFKC_CF"
34 };
35
UnicodeTest()36 UnicodeTest::UnicodeTest()
37 {
38 UErrorCode errorCode=U_ZERO_ERROR;
39 unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
40 if(U_FAILURE(errorCode)) {
41 delete unknownPropertyNames;
42 unknownPropertyNames=NULL;
43 }
44 // Ignore some property names altogether.
45 for(int32_t i=0; i<UPRV_LENGTHOF(ignorePropNames); ++i) {
46 unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
47 }
48 }
49
~UnicodeTest()50 UnicodeTest::~UnicodeTest()
51 {
52 delete unknownPropertyNames;
53 }
54
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)55 void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
56 {
57 if(exec) {
58 logln("TestSuite UnicodeTest: ");
59 }
60 TESTCASE_AUTO_BEGIN;
61 TESTCASE_AUTO(TestAdditionalProperties);
62 TESTCASE_AUTO(TestBinaryValues);
63 TESTCASE_AUTO(TestConsistency);
64 TESTCASE_AUTO(TestPatternProperties);
65 TESTCASE_AUTO(TestScriptMetadata);
66 TESTCASE_AUTO(TestBidiPairedBracketType);
67 TESTCASE_AUTO(TestEmojiProperties);
68 TESTCASE_AUTO(TestIndicPositionalCategory);
69 TESTCASE_AUTO(TestIndicSyllabicCategory);
70 TESTCASE_AUTO(TestVerticalOrientation);
71 TESTCASE_AUTO(TestDefaultScriptExtensions);
72 TESTCASE_AUTO(TestInvalidCodePointFolding);
73 #if !UCONFIG_NO_NORMALIZATION
74 TESTCASE_AUTO(TestBinaryCharacterProperties);
75 TESTCASE_AUTO(TestIntCharacterProperties);
76 #endif
77 TESTCASE_AUTO_END;
78 }
79
80 //====================================================
81 // private data used by the tests
82 //====================================================
83
84 // test DerivedCoreProperties.txt -------------------------------------------
85
86 // copied from genprops.c
87 static int32_t
getTokenIndex(const char * const tokens[],int32_t countTokens,const char * s)88 getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
89 const char *t, *z;
90 int32_t i, j;
91
92 s=u_skipWhitespace(s);
93 for(i=0; i<countTokens; ++i) {
94 t=tokens[i];
95 if(t!=NULL) {
96 for(j=0;; ++j) {
97 if(t[j]!=0) {
98 if(s[j]!=t[j]) {
99 break;
100 }
101 } else {
102 z=u_skipWhitespace(s+j);
103 if(*z==';' || *z==0) {
104 return i;
105 } else {
106 break;
107 }
108 }
109 }
110 }
111 }
112 return -1;
113 }
114
115 static const char *const
116 derivedPropsNames[]={
117 "Math",
118 "Alphabetic",
119 "Lowercase",
120 "Uppercase",
121 "ID_Start",
122 "ID_Continue",
123 "XID_Start",
124 "XID_Continue",
125 "Default_Ignorable_Code_Point",
126 "Full_Composition_Exclusion",
127 "Grapheme_Extend",
128 "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
129 "Grapheme_Base",
130 "Cased",
131 "Case_Ignorable",
132 "Changes_When_Lowercased",
133 "Changes_When_Uppercased",
134 "Changes_When_Titlecased",
135 "Changes_When_Casefolded",
136 "Changes_When_Casemapped",
137 "Changes_When_NFKC_Casefolded"
138 };
139
140 static const UProperty
141 derivedPropsIndex[]={
142 UCHAR_MATH,
143 UCHAR_ALPHABETIC,
144 UCHAR_LOWERCASE,
145 UCHAR_UPPERCASE,
146 UCHAR_ID_START,
147 UCHAR_ID_CONTINUE,
148 UCHAR_XID_START,
149 UCHAR_XID_CONTINUE,
150 UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
151 UCHAR_FULL_COMPOSITION_EXCLUSION,
152 UCHAR_GRAPHEME_EXTEND,
153 UCHAR_GRAPHEME_LINK,
154 UCHAR_GRAPHEME_BASE,
155 UCHAR_CASED,
156 UCHAR_CASE_IGNORABLE,
157 UCHAR_CHANGES_WHEN_LOWERCASED,
158 UCHAR_CHANGES_WHEN_UPPERCASED,
159 UCHAR_CHANGES_WHEN_TITLECASED,
160 UCHAR_CHANGES_WHEN_CASEFOLDED,
161 UCHAR_CHANGES_WHEN_CASEMAPPED,
162 UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
163 };
164
165 static int32_t numErrors[UPRV_LENGTHOF(derivedPropsIndex)]={ 0 };
166
167 enum { MAX_ERRORS=50 };
168
169 U_CFUNC void U_CALLCONV
derivedPropsLineFn(void * context,char * fields[][2],int32_t,UErrorCode * pErrorCode)170 derivedPropsLineFn(void *context,
171 char *fields[][2], int32_t /* fieldCount */,
172 UErrorCode *pErrorCode)
173 {
174 UnicodeTest *me=(UnicodeTest *)context;
175 uint32_t start, end;
176 int32_t i;
177
178 u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
179 if(U_FAILURE(*pErrorCode)) {
180 me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
181 return;
182 }
183
184 /* parse derived binary property name, ignore unknown names */
185 i=getTokenIndex(derivedPropsNames, UPRV_LENGTHOF(derivedPropsNames), fields[1][0]);
186 if(i<0) {
187 UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
188 propName.trim();
189 if(me->unknownPropertyNames->find(propName)==NULL) {
190 UErrorCode errorCode=U_ZERO_ERROR;
191 me->unknownPropertyNames->puti(propName, 1, errorCode);
192 me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
193 }
194 return;
195 }
196
197 me->derivedProps[i].add(start, end);
198 }
199
TestAdditionalProperties()200 void UnicodeTest::TestAdditionalProperties() {
201 #if !UCONFIG_NO_NORMALIZATION
202 // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
203 if(UPRV_LENGTHOF(derivedProps)<UPRV_LENGTHOF(derivedPropsNames)) {
204 errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
205 UPRV_LENGTHOF(derivedPropsNames));
206 return;
207 }
208 if(UPRV_LENGTHOF(derivedPropsIndex)!=UPRV_LENGTHOF(derivedPropsNames)) {
209 errln("error in ucdtest.cpp: UPRV_LENGTHOF(derivedPropsIndex)!=UPRV_LENGTHOF(derivedPropsNames)\n");
210 return;
211 }
212
213 char path[500];
214 if(getUnidataPath(path) == NULL) {
215 errln("unable to find path to source/data/unidata/");
216 return;
217 }
218 char *basename=strchr(path, 0);
219 strcpy(basename, "DerivedCoreProperties.txt");
220
221 char *fields[2][2];
222 UErrorCode errorCode=U_ZERO_ERROR;
223 u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
224 if(U_FAILURE(errorCode)) {
225 errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
226 return;
227 }
228
229 strcpy(basename, "DerivedNormalizationProps.txt");
230 u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
231 if(U_FAILURE(errorCode)) {
232 errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
233 return;
234 }
235
236 // now we have all derived core properties in the UnicodeSets
237 // run them all through the API
238 int32_t rangeCount, range;
239 uint32_t i;
240 UChar32 start, end;
241
242 // test all TRUE properties
243 for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
244 rangeCount=derivedProps[i].getRangeCount();
245 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
246 start=derivedProps[i].getRangeStart(range);
247 end=derivedProps[i].getRangeEnd(range);
248 for(; start<=end; ++start) {
249 if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
250 dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
251 if(++numErrors[i]>=MAX_ERRORS) {
252 dataerrln("Too many errors, moving to the next test");
253 break;
254 }
255 }
256 }
257 }
258 }
259
260 // invert all properties
261 for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
262 derivedProps[i].complement();
263 }
264
265 // test all FALSE properties
266 for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
267 rangeCount=derivedProps[i].getRangeCount();
268 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
269 start=derivedProps[i].getRangeStart(range);
270 end=derivedProps[i].getRangeEnd(range);
271 for(; start<=end; ++start) {
272 if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
273 errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
274 if(++numErrors[i]>=MAX_ERRORS) {
275 errln("Too many errors, moving to the next test");
276 break;
277 }
278 }
279 }
280 }
281 }
282 #endif /* !UCONFIG_NO_NORMALIZATION */
283 }
284
TestBinaryValues()285 void UnicodeTest::TestBinaryValues() {
286 /*
287 * Unicode 5.1 explicitly defines binary property value aliases.
288 * Verify that they are all recognized.
289 */
290 UErrorCode errorCode=U_ZERO_ERROR;
291 UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
292 if(U_FAILURE(errorCode)) {
293 dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
294 return;
295 }
296
297 static const char *const falseValues[]={ "N", "No", "F", "False" };
298 static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
299 int32_t i;
300 for(i=0; i<UPRV_LENGTHOF(falseValues); ++i) {
301 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
302 pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
303 errorCode=U_ZERO_ERROR;
304 UnicodeSet set(pattern, errorCode);
305 if(U_FAILURE(errorCode)) {
306 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
307 continue;
308 }
309 set.complement();
310 if(set!=alpha) {
311 errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
312 }
313 }
314 for(i=0; i<UPRV_LENGTHOF(trueValues); ++i) {
315 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
316 pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
317 errorCode=U_ZERO_ERROR;
318 UnicodeSet set(pattern, errorCode);
319 if(U_FAILURE(errorCode)) {
320 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
321 continue;
322 }
323 if(set!=alpha) {
324 errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
325 }
326 }
327 }
328
TestConsistency()329 void UnicodeTest::TestConsistency() {
330 #if !UCONFIG_NO_NORMALIZATION
331 /*
332 * Test for an example that getCanonStartSet() delivers
333 * all characters that compose from the input one,
334 * even in multiple steps.
335 * For example, the set for "I" (0049) should contain both
336 * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
337 * In general, the set for the middle such character should be a subset
338 * of the set for the first.
339 */
340 IcuTestErrorCode errorCode(*this, "TestConsistency");
341 const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
342 const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
343 if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
344 dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
345 errorCode.errorName());
346 errorCode.reset();
347 return;
348 }
349
350 UnicodeSet set1, set2;
351 if (nfcImpl->getCanonStartSet(0x49, set1)) {
352 /* enumerate all characters that are plausible to be latin letters */
353 for(UChar start=0xa0; start<0x2000; ++start) {
354 UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
355 if(decomp.length()>1 && decomp[0]==0x49) {
356 set2.add(start);
357 }
358 }
359
360 if (set1!=set2) {
361 errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
362 }
363 // This was available in cucdtst.c but the test had to move to intltest
364 // because the new internal normalization functions are in C++.
365 //compareUSets(set1, set2,
366 // "[canon start set of 0049]", "[all c with canon decomp with 0049]",
367 // TRUE);
368 } else {
369 errln("NFC.getCanonStartSet() returned FALSE");
370 }
371 #endif
372 }
373
374 /**
375 * Test various implementations of Pattern_Syntax & Pattern_White_Space.
376 */
TestPatternProperties()377 void UnicodeTest::TestPatternProperties() {
378 IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
379 UnicodeSet syn_pp;
380 UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
381 UnicodeSet syn_list(
382 "[!-/\\:-@\\[-\\^`\\{-~"
383 "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
384 "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
385 "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
386 UnicodeSet ws_pp;
387 UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
388 UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
389 UnicodeSet syn_ws_pp;
390 UnicodeSet syn_ws_prop(syn_prop);
391 syn_ws_prop.addAll(ws_prop);
392 for(UChar32 c=0; c<=0xffff; ++c) {
393 if(PatternProps::isSyntax(c)) {
394 syn_pp.add(c);
395 }
396 if(PatternProps::isWhiteSpace(c)) {
397 ws_pp.add(c);
398 }
399 if(PatternProps::isSyntaxOrWhiteSpace(c)) {
400 syn_ws_pp.add(c);
401 }
402 }
403 compareUSets(syn_pp, syn_prop,
404 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
405 compareUSets(syn_pp, syn_list,
406 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
407 compareUSets(ws_pp, ws_prop,
408 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
409 compareUSets(ws_pp, ws_list,
410 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
411 compareUSets(syn_ws_pp, syn_ws_prop,
412 "PatternProps.isSyntaxOrWhiteSpace()",
413 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
414 }
415
416 // So far only minimal port of Java & cucdtst.c compareUSets().
417 UBool
compareUSets(const UnicodeSet & a,const UnicodeSet & b,const char * a_name,const char * b_name,UBool diffIsError)418 UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
419 const char *a_name, const char *b_name,
420 UBool diffIsError) {
421 UBool same= a==b;
422 if(!same && diffIsError) {
423 errln("Sets are different: %s vs. %s\n", a_name, b_name);
424 }
425 return same;
426 }
427
428 namespace {
429
430 /**
431 * Maps a special script code to the most common script of its encoded characters.
432 */
getCharScript(UScriptCode script)433 UScriptCode getCharScript(UScriptCode script) {
434 switch(script) {
435 case USCRIPT_HAN_WITH_BOPOMOFO:
436 case USCRIPT_SIMPLIFIED_HAN:
437 case USCRIPT_TRADITIONAL_HAN:
438 return USCRIPT_HAN;
439 case USCRIPT_JAPANESE:
440 return USCRIPT_HIRAGANA;
441 case USCRIPT_JAMO:
442 case USCRIPT_KOREAN:
443 return USCRIPT_HANGUL;
444 case USCRIPT_SYMBOLS_EMOJI:
445 return USCRIPT_SYMBOLS;
446 default:
447 return script;
448 }
449 }
450
451 } // namespace
452
TestScriptMetadata()453 void UnicodeTest::TestScriptMetadata() {
454 IcuTestErrorCode errorCode(*this, "TestScriptMetadata()");
455 UnicodeSet rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode);
456 // So far, sample characters are uppercase.
457 // Georgian is special.
458 UnicodeSet cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode);
459 for(int32_t sci = 0; sci < USCRIPT_CODE_LIMIT; ++sci) {
460 UScriptCode sc = (UScriptCode)sci;
461 // Run the test with -v to see which script has failures:
462 // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 6 FAIL
463 logln(uscript_getShortName(sc));
464 UScriptUsage usage = uscript_getUsage(sc);
465 UnicodeString sample = uscript_getSampleUnicodeString(sc);
466 UnicodeSet scriptSet;
467 scriptSet.applyIntPropertyValue(UCHAR_SCRIPT, sc, errorCode);
468 if(usage == USCRIPT_USAGE_NOT_ENCODED) {
469 assertTrue("not encoded, no sample", sample.isEmpty());
470 assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc));
471 assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc));
472 assertFalse("not encoded, not cased", uscript_isCased(sc));
473 assertTrue("not encoded, no characters", scriptSet.isEmpty());
474 } else {
475 assertFalse("encoded, has a sample character", sample.isEmpty());
476 UChar32 firstChar = sample.char32At(0);
477 UScriptCode charScript = getCharScript(sc);
478 assertEquals("script(sample(script))",
479 (int32_t)charScript, (int32_t)uscript_getScript(firstChar, errorCode));
480 assertEquals("RTL vs. set", (UBool)rtl.contains(firstChar), (UBool)uscript_isRightToLeft(sc));
481 assertEquals("cased vs. set", (UBool)cased.contains(firstChar), (UBool)uscript_isCased(sc));
482 assertEquals("encoded, has characters", (UBool)(sc == charScript), (UBool)(!scriptSet.isEmpty()));
483 if(uscript_isRightToLeft(sc)) {
484 rtl.removeAll(scriptSet);
485 }
486 if(uscript_isCased(sc)) {
487 cased.removeAll(scriptSet);
488 }
489 }
490 }
491 UnicodeString pattern;
492 assertEquals("no remaining RTL characters",
493 UnicodeString("[]"), rtl.toPattern(pattern));
494 assertEquals("no remaining cased characters",
495 UnicodeString("[]"), cased.toPattern(pattern));
496
497 assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN));
498 assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI));
499 assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN));
500 }
501
TestBidiPairedBracketType()502 void UnicodeTest::TestBidiPairedBracketType() {
503 // BidiBrackets-6.3.0.txt says:
504 //
505 // The set of code points listed in this file was originally derived
506 // using the character properties General_Category (gc), Bidi_Class (bc),
507 // Bidi_Mirrored (Bidi_M), and Bidi_Mirroring_Glyph (bmg), as follows:
508 // two characters, A and B, form a pair if A has gc=Ps and B has gc=Pe,
509 // both have bc=ON and Bidi_M=Y, and bmg of A is B. Bidi_Paired_Bracket
510 // maps A to B and vice versa, and their Bidi_Paired_Bracket_Type
511 // property values are Open and Close, respectively.
512 IcuTestErrorCode errorCode(*this, "TestBidiPairedBracketType()");
513 UnicodeSet bpt("[:^bpt=n:]", errorCode);
514 assertTrue("bpt!=None is not empty", !bpt.isEmpty());
515 // The following should always be true.
516 UnicodeSet mirrored("[:Bidi_M:]", errorCode);
517 UnicodeSet other_neutral("[:bc=ON:]", errorCode);
518 assertTrue("bpt!=None is a subset of Bidi_M", mirrored.containsAll(bpt));
519 assertTrue("bpt!=None is a subset of bc=ON", other_neutral.containsAll(bpt));
520 // The following are true at least initially in Unicode 6.3.
521 UnicodeSet bpt_open("[:bpt=o:]", errorCode);
522 UnicodeSet bpt_close("[:bpt=c:]", errorCode);
523 UnicodeSet ps("[:Ps:]", errorCode);
524 UnicodeSet pe("[:Pe:]", errorCode);
525 assertTrue("bpt=Open is a subset of Ps", ps.containsAll(bpt_open));
526 assertTrue("bpt=Close is a subset of Pe", pe.containsAll(bpt_close));
527 }
528
TestEmojiProperties()529 void UnicodeTest::TestEmojiProperties() {
530 assertFalse("space is not Emoji", u_hasBinaryProperty(0x20, UCHAR_EMOJI));
531 assertTrue("shooting star is Emoji", u_hasBinaryProperty(0x1F320, UCHAR_EMOJI));
532 IcuTestErrorCode errorCode(*this, "TestEmojiProperties()");
533 UnicodeSet emoji("[:Emoji:]", errorCode);
534 assertTrue("lots of Emoji", emoji.size() > 700);
535
536 assertTrue("shooting star is Emoji_Presentation",
537 u_hasBinaryProperty(0x1F320, UCHAR_EMOJI_PRESENTATION));
538 assertTrue("Fitzpatrick 6 is Emoji_Modifier",
539 u_hasBinaryProperty(0x1F3FF, UCHAR_EMOJI_MODIFIER));
540 assertTrue("happy person is Emoji_Modifier_Base",
541 u_hasBinaryProperty(0x1F64B, UCHAR_EMOJI_MODIFIER_BASE));
542 assertTrue("asterisk is Emoji_Component",
543 u_hasBinaryProperty(0x2A, UCHAR_EMOJI_COMPONENT));
544 assertTrue("copyright is Extended_Pictographic",
545 u_hasBinaryProperty(0xA9, UCHAR_EXTENDED_PICTOGRAPHIC));
546 }
547
TestIndicPositionalCategory()548 void UnicodeTest::TestIndicPositionalCategory() {
549 IcuTestErrorCode errorCode(*this, "TestIndicPositionalCategory()");
550 UnicodeSet na(u"[:InPC=NA:]", errorCode);
551 assertTrue("mostly NA", 1000000 <= na.size() && na.size() <= UCHAR_MAX_VALUE - 500);
552 UnicodeSet vol(u"[:InPC=Visual_Order_Left:]", errorCode);
553 assertTrue("some Visual_Order_Left", 19 <= vol.size() && vol.size() <= 100);
554 assertEquals("U+08FF: NA", U_INPC_NA,
555 u_getIntPropertyValue(0x08FF, UCHAR_INDIC_POSITIONAL_CATEGORY));
556 assertEquals("U+0900: Top", U_INPC_TOP,
557 u_getIntPropertyValue(0x0900, UCHAR_INDIC_POSITIONAL_CATEGORY));
558 assertEquals("U+10A06: Overstruck", U_INPC_OVERSTRUCK,
559 u_getIntPropertyValue(0x10A06, UCHAR_INDIC_POSITIONAL_CATEGORY));
560 }
561
TestIndicSyllabicCategory()562 void UnicodeTest::TestIndicSyllabicCategory() {
563 IcuTestErrorCode errorCode(*this, "TestIndicSyllabicCategory()");
564 UnicodeSet other(u"[:InSC=Other:]", errorCode);
565 assertTrue("mostly Other", 1000000 <= other.size() && other.size() <= UCHAR_MAX_VALUE - 500);
566 UnicodeSet ava(u"[:InSC=Avagraha:]", errorCode);
567 assertTrue("some Avagraha", 16 <= ava.size() && ava.size() <= 100);
568 assertEquals("U+08FF: Other", U_INSC_OTHER,
569 u_getIntPropertyValue(0x08FF, UCHAR_INDIC_SYLLABIC_CATEGORY));
570 assertEquals("U+0900: Bindu", U_INSC_BINDU,
571 u_getIntPropertyValue(0x0900, UCHAR_INDIC_SYLLABIC_CATEGORY));
572 assertEquals("U+11065: Brahmi_Joining_Number", U_INSC_BRAHMI_JOINING_NUMBER,
573 u_getIntPropertyValue(0x11065, UCHAR_INDIC_SYLLABIC_CATEGORY));
574 }
575
TestVerticalOrientation()576 void UnicodeTest::TestVerticalOrientation() {
577 IcuTestErrorCode errorCode(*this, "TestVerticalOrientation()");
578 UnicodeSet r(u"[:vo=R:]", errorCode);
579 assertTrue("mostly R", 0xc0000 <= r.size() && r.size() <= 0xd0000);
580 UnicodeSet u(u"[:vo=U:]", errorCode);
581 assertTrue("much U", 0x40000 <= u.size() && u.size() <= 0x50000);
582 UnicodeSet tu(u"[:vo=Tu:]", errorCode);
583 assertTrue("some Tu", 147 <= tu.size() && tu.size() <= 300);
584 assertEquals("U+0E01: Rotated", U_VO_ROTATED,
585 u_getIntPropertyValue(0x0E01, UCHAR_VERTICAL_ORIENTATION));
586 assertEquals("U+3008: Transformed_Rotated", U_VO_TRANSFORMED_ROTATED,
587 u_getIntPropertyValue(0x3008, UCHAR_VERTICAL_ORIENTATION));
588 assertEquals("U+33333: Upright", U_VO_UPRIGHT,
589 u_getIntPropertyValue(0x33333, UCHAR_VERTICAL_ORIENTATION));
590 }
591
TestDefaultScriptExtensions()592 void UnicodeTest::TestDefaultScriptExtensions() {
593 // Block 3000..303F CJK Symbols and Punctuation defaults to scx=Bopo Hang Hani Hira Kana Yiii
594 // but some of its characters revert to scx=<script> which is usually Common.
595 IcuTestErrorCode errorCode(*this, "TestDefaultScriptExtensions()");
596 UScriptCode scx[20];
597 scx[0] = USCRIPT_INVALID_CODE;
598 assertEquals("U+3000 num scx", 1, // IDEOGRAPHIC SPACE
599 uscript_getScriptExtensions(0x3000, scx, UPRV_LENGTHOF(scx), errorCode));
600 assertEquals("U+3000 num scx[0]", USCRIPT_COMMON, scx[0]);
601 scx[0] = USCRIPT_INVALID_CODE;
602 assertEquals("U+3012 num scx", 1, // POSTAL MARK
603 uscript_getScriptExtensions(0x3012, scx, UPRV_LENGTHOF(scx), errorCode));
604 assertEquals("U+3012 num scx[0]", USCRIPT_COMMON, scx[0]);
605 }
606
TestInvalidCodePointFolding(void)607 void UnicodeTest::TestInvalidCodePointFolding(void) {
608 // Test behavior when an invalid code point is passed to u_foldCase
609 static const UChar32 invalidCodePoints[] = {
610 0xD800, // lead surrogate
611 0xDFFF, // trail surrogate
612 0xFDD0, // noncharacter
613 0xFFFF, // noncharacter
614 0x110000, // out of range
615 -1 // negative
616 };
617 for (int32_t i=0; i<UPRV_LENGTHOF(invalidCodePoints); ++i) {
618 UChar32 cp = invalidCodePoints[i];
619 assertEquals("Invalid code points should be echoed back",
620 cp, u_foldCase(cp, U_FOLD_CASE_DEFAULT));
621 assertEquals("Invalid code points should be echoed back",
622 cp, u_foldCase(cp, U_FOLD_CASE_EXCLUDE_SPECIAL_I));
623 }
624 }
625
TestBinaryCharacterProperties()626 void UnicodeTest::TestBinaryCharacterProperties() {
627 #if !UCONFIG_NO_NORMALIZATION
628 IcuTestErrorCode errorCode(*this, "TestBinaryCharacterProperties()");
629 // Spot-check getBinaryPropertySet() vs. hasBinaryProperty().
630 for (int32_t prop = 0; prop < UCHAR_BINARY_LIMIT; ++prop) {
631 const USet *uset = u_getBinaryPropertySet((UProperty)prop, errorCode);
632 if (errorCode.errIfFailureAndReset("u_getBinaryPropertySet(%d)", (int)prop)) {
633 continue;
634 }
635 const UnicodeSet &set = *UnicodeSet::fromUSet(uset);
636 int32_t size = set.size();
637 if (size == 0) {
638 assertFalse(UnicodeString("!hasBinaryProperty(U+0020, ") + prop + u")",
639 u_hasBinaryProperty(0x20, (UProperty)prop));
640 assertFalse(UnicodeString("!hasBinaryProperty(U+0061, ") + prop + u")",
641 u_hasBinaryProperty(0x61, (UProperty)prop));
642 assertFalse(UnicodeString("!hasBinaryProperty(U+4E00, ") + prop + u")",
643 u_hasBinaryProperty(0x4e00, (UProperty)prop));
644 } else {
645 UChar32 c = set.charAt(0);
646 if (c > 0) {
647 assertFalse(
648 UnicodeString("!hasBinaryProperty(") + TestUtility::hex(c - 1) +
649 u", " + prop + u")",
650 u_hasBinaryProperty(c - 1, (UProperty)prop));
651 }
652 assertTrue(
653 UnicodeString("hasBinaryProperty(") + TestUtility::hex(c) +
654 u", " + prop + u")",
655 u_hasBinaryProperty(c, (UProperty)prop));
656 c = set.charAt(size - 1);
657 assertTrue(
658 UnicodeString("hasBinaryProperty(") + TestUtility::hex(c) +
659 u", " + prop + u")",
660 u_hasBinaryProperty(c, (UProperty)prop));
661 if (c < 0x10ffff) {
662 assertFalse(
663 UnicodeString("!hasBinaryProperty(") + TestUtility::hex(c + 1) +
664 u", " + prop + u")",
665 u_hasBinaryProperty(c + 1, (UProperty)prop));
666 }
667 }
668 }
669 #endif
670 }
671
TestIntCharacterProperties()672 void UnicodeTest::TestIntCharacterProperties() {
673 #if !UCONFIG_NO_NORMALIZATION
674 IcuTestErrorCode errorCode(*this, "TestIntCharacterProperties()");
675 // Spot-check getIntPropertyMap() vs. getIntPropertyValue().
676 for (int32_t prop = UCHAR_INT_START; prop < UCHAR_INT_LIMIT; ++prop) {
677 const UCPMap *map = u_getIntPropertyMap((UProperty)prop, errorCode);
678 if (errorCode.errIfFailureAndReset("u_getIntPropertyMap(%d)", (int)prop)) {
679 continue;
680 }
681 uint32_t value;
682 UChar32 end = ucpmap_getRange(map, 0, UCPMAP_RANGE_NORMAL, 0, nullptr, nullptr, &value);
683 assertTrue("int property first range", end >= 0);
684 UChar32 c = end / 2;
685 assertEquals(UnicodeString("int property first range value at ") + TestUtility::hex(c),
686 u_getIntPropertyValue(c, (UProperty)prop), value);
687 end = ucpmap_getRange(map, 0x5000, UCPMAP_RANGE_NORMAL, 0, nullptr, nullptr, &value);
688 assertTrue("int property later range", end >= 0);
689 assertEquals(UnicodeString("int property later range value at ") + TestUtility::hex(end),
690 u_getIntPropertyValue(end, (UProperty)prop), value);
691 // ucpmap_get() API coverage
692 // TODO: move to cucdtst.c
693 assertEquals(
694 "int property upcmap_get(U+0061)",
695 u_getIntPropertyValue(0x61, (UProperty)prop), ucpmap_get(map, 0x61));
696 }
697 #endif
698 }
699