1 /*
2 **********************************************************************
3 * Copyright (C) 1999-2012, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 **********************************************************************
6 * Date Name Description
7 * 12/09/99 aliu Ported from Java.
8 **********************************************************************
9 */
10
11 #include "unicode/utypes.h"
12
13 #if !UCONFIG_NO_COLLATION
14
15 #include "thcoll.h"
16 #include "unicode/utypes.h"
17 #include "unicode/coll.h"
18 #include "unicode/localpointer.h"
19 #include "unicode/sortkey.h"
20 #include "unicode/ustring.h"
21 #include "cstring.h"
22 #include "filestrm.h"
23 #include "textfile.h"
24
25 /**
26 * The TestDictionary test expects a file of this name, with this
27 * encoding, to be present in the directory $ICU/source/test/testdata.
28 */
29 //#define TEST_FILE "th18057.txt"
30
31 /**
32 * This is the most failures we show in TestDictionary. If this number
33 * is < 0, we show all failures.
34 */
35 #define MAX_FAILURES_TO_SHOW -1
36
CollationThaiTest()37 CollationThaiTest::CollationThaiTest() {
38 UErrorCode status = U_ZERO_ERROR;
39 coll = Collator::createInstance(Locale("th", "TH", ""), status);
40 if (coll && U_SUCCESS(status)) {
41 //coll->setStrength(Collator::TERTIARY);
42 } else {
43 delete coll;
44 coll = 0;
45 }
46 }
47
~CollationThaiTest()48 CollationThaiTest::~CollationThaiTest() {
49 delete coll;
50 }
51
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)52 void CollationThaiTest::runIndexedTest(int32_t index, UBool exec, const char* &name,
53 char* /*par*/) {
54
55 if((!coll) && exec) {
56 dataerrln(__FILE__ " cannot test - failed to create collator.");
57 name = "some test";
58 return;
59 }
60
61 switch (index) {
62 TESTCASE(0,TestDictionary);
63 TESTCASE(1,TestCornerCases);
64 TESTCASE(2,TestNamesList);
65 TESTCASE(3,TestInvalidThai);
66 TESTCASE(4,TestReordering);
67 default: name = ""; break;
68 }
69 }
70
71 /**
72 * Read the external names list, and confirms that the collator
73 * gets the same results when comparing lines one to another
74 * using regular and iterative comparison.
75 */
TestNamesList(void)76 void CollationThaiTest::TestNamesList(void) {
77 if (coll == 0) {
78 errln("Error: could not construct Thai collator");
79 return;
80 }
81
82 UErrorCode ec = U_ZERO_ERROR;
83 TextFile names("TestNames_Thai.txt", "UTF16LE", ec);
84 if (U_FAILURE(ec)) {
85 logln("Can't open TestNames_Thai.txt: %s; skipping test",
86 u_errorName(ec));
87 return;
88 }
89
90 //
91 // Loop through each word in the dictionary and compare it to the previous
92 // word. They should be in sorted order.
93 //
94 UnicodeString lastWord, word;
95 //int32_t failed = 0;
96 int32_t wordCount = 0;
97 while (names.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) {
98
99 // Show the first 8 words being compared, so we can see what's happening
100 ++wordCount;
101 if (wordCount <= 8) {
102 UnicodeString str;
103 logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str));
104 }
105
106 if (lastWord.length() > 0) {
107 Collator::EComparisonResult result = coll->compare(lastWord, word);
108 doTest(coll, lastWord, word, result);
109 }
110 lastWord = word;
111 }
112
113 assertSuccess("readLine", ec);
114
115 logln((UnicodeString)"Words checked: " + wordCount);
116 }
117
118 /**
119 * Read the external dictionary file, which is already in proper
120 * sorted order, and confirm that the collator compares each line as
121 * preceding the following line.
122 */
TestDictionary(void)123 void CollationThaiTest::TestDictionary(void) {
124 if (coll == 0) {
125 errln("Error: could not construct Thai collator");
126 return;
127 }
128
129 UErrorCode ec = U_ZERO_ERROR;
130 TextFile riwords("riwords.txt", "UTF8", ec);
131 if (U_FAILURE(ec)) {
132 logln("Can't open riwords.txt: %s; skipping test",
133 u_errorName(ec));
134 return;
135 }
136
137 //
138 // Loop through each word in the dictionary and compare it to the previous
139 // word. They should be in sorted order.
140 //
141 UnicodeString lastWord, word;
142 int32_t failed = 0;
143 int32_t wordCount = 0;
144 while (riwords.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) {
145
146 // Show the first 8 words being compared, so we can see what's happening
147 ++wordCount;
148 if (wordCount <= 8) {
149 UnicodeString str;
150 logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str));
151 }
152
153 if (lastWord.length() > 0) {
154 int32_t result = coll->compare(lastWord, word);
155
156 if (result > 0) {
157 failed++;
158 if (MAX_FAILURES_TO_SHOW < 0 || failed <= MAX_FAILURES_TO_SHOW) {
159 UnicodeString str;
160 UnicodeString msg =
161 UnicodeString("--------------------------------------------\n")
162 + riwords.getLineNumber()
163 + " compare(" + IntlTest::prettify(lastWord, str);
164 msg += UnicodeString(", ")
165 + IntlTest::prettify(word, str) + ") returned " + result
166 + ", expected -1\n";
167 UErrorCode status = U_ZERO_ERROR;
168 CollationKey k1, k2;
169 coll->getCollationKey(lastWord, k1, status);
170 coll->getCollationKey(word, k2, status);
171 if (U_FAILURE(status)) {
172 errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
173 return;
174 }
175 msg.append("key1: ").append(prettify(k1, str)).append("\n");
176 msg.append("key2: ").append(prettify(k2, str));
177 errln(msg);
178 }
179 }
180 }
181 lastWord = word;
182 }
183
184 assertSuccess("readLine", ec);
185
186 if (failed != 0) {
187 if (failed > MAX_FAILURES_TO_SHOW) {
188 errln((UnicodeString)"Too many failures; only the first " +
189 MAX_FAILURES_TO_SHOW + " failures were shown");
190 }
191 errln((UnicodeString)"Summary: " + failed + " of " + (riwords.getLineNumber() - 1) +
192 " comparisons failed");
193 }
194
195 logln((UnicodeString)"Words checked: " + wordCount);
196 }
197
198 /**
199 * Odd corner conditions taken from "How to Sort Thai Without Rewriting Sort",
200 * by Doug Cooper, http://seasrc.th.net/paper/thaisort.zip
201 */
TestCornerCases(void)202 void CollationThaiTest::TestCornerCases(void) {
203 const char* TESTS[] = {
204 // Shorter words precede longer
205 "\\u0e01", "<", "\\u0e01\\u0e01",
206
207 // Tone marks are considered after letters (i.e. are primary ignorable)
208 "\\u0e01\\u0e32", "<", "\\u0e01\\u0e49\\u0e32",
209
210 // ditto for other over-marks
211 "\\u0e01\\u0e32", "<", "\\u0e01\\u0e32\\u0e4c",
212
213 // commonly used mark-in-context order.
214 // In effect, marks are sorted after each syllable.
215 "\\u0e01\\u0e32\\u0e01\\u0e49\\u0e32", "<", "\\u0e01\\u0e48\\u0e32\\u0e01\\u0e49\\u0e32",
216
217 // Hyphens and other punctuation follow whitespace but come before letters
218 "\\u0e01\\u0e32", "=", "\\u0e01\\u0e32-",
219 "\\u0e01\\u0e32-", "<", "\\u0e01\\u0e32\\u0e01\\u0e32",
220
221 // Doubler follows an indentical word without the doubler
222 "\\u0e01\\u0e32", "=", "\\u0e01\\u0e32\\u0e46",
223 "\\u0e01\\u0e32\\u0e46", "<", "\\u0e01\\u0e32\\u0e01\\u0e32",
224
225
226 // \\u0e45 after either \\u0e24 or \\u0e26 is treated as a single
227 // combining character, similar to "c < ch" in traditional spanish.
228 // TODO: beef up this case
229 "\\u0e24\\u0e29\\u0e35", "<", "\\u0e24\\u0e45\\u0e29\\u0e35",
230 "\\u0e26\\u0e29\\u0e35", "<", "\\u0e26\\u0e45\\u0e29\\u0e35",
231
232 // Vowels reorder, should compare \\u0e2d and \\u0e34
233 "\\u0e40\\u0e01\\u0e2d", "<", "\\u0e40\\u0e01\\u0e34",
234
235 // Tones are compared after the rest of the word (e.g. primary ignorable)
236 "\\u0e01\\u0e32\\u0e01\\u0e48\\u0e32", "<", "\\u0e01\\u0e49\\u0e32\\u0e01\\u0e32",
237
238 // Periods are ignored entirely
239 "\\u0e01.\\u0e01.", "<", "\\u0e01\\u0e32",
240 };
241 const int32_t TESTS_length = (int32_t)(sizeof(TESTS)/sizeof(TESTS[0]));
242
243 if (coll == 0) {
244 errln("Error: could not construct Thai collator");
245 return;
246 }
247 compareArray(*coll, TESTS, TESTS_length);
248 }
249
250 //------------------------------------------------------------------------
251 // Internal utilities
252 //------------------------------------------------------------------------
253
compareArray(Collator & c,const char * tests[],int32_t testsLength)254 void CollationThaiTest::compareArray(Collator& c, const char* tests[],
255 int32_t testsLength) {
256 for (int32_t i = 0; i < testsLength; i += 3) {
257
258 Collator::EComparisonResult expect;
259 if (tests[i+1][0] == '<') {
260 expect = Collator::LESS;
261 } else if (tests[i+1][0] == '>') {
262 expect = Collator::GREATER;
263 } else if (tests[i+1][0] == '=') {
264 expect = Collator::EQUAL;
265 } else {
266 // expect = Integer.decode(tests[i+1]).intValue();
267 errln((UnicodeString)"Error: unknown operator " + tests[i+1]);
268 return;
269 }
270
271 UnicodeString s1, s2;
272 parseChars(s1, tests[i]);
273 parseChars(s2, tests[i+2]);
274
275 doTest(&c, s1, s2, expect);
276 #if 0
277 UErrorCode status = U_ZERO_ERROR;
278 int32_t result = c.compare(s1, s2);
279 if (sign(result) != sign(expect))
280 {
281 UnicodeString t1, t2;
282 errln(UnicodeString("") +
283 i/3 + ": compare(" + IntlTest::prettify(s1, t1)
284 + " , " + IntlTest::prettify(s2, t2)
285 + ") got " + result + "; expected " + expect);
286
287 CollationKey k1, k2;
288 c.getCollationKey(s1, k1, status);
289 c.getCollationKey(s2, k2, status);
290 if (U_FAILURE(status)) {
291 errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
292 return;
293 }
294 errln((UnicodeString)" key1: " + prettify(k1, t1) );
295 errln((UnicodeString)" key2: " + prettify(k2, t2) );
296 }
297 else
298 {
299 // Collator.compare worked OK; now try the collation keys
300 CollationKey k1, k2;
301 c.getCollationKey(s1, k1, status);
302 c.getCollationKey(s2, k2, status);
303 if (U_FAILURE(status)) {
304 errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
305 return;
306 }
307
308 result = k1.compareTo(k2);
309 if (sign(result) != sign(expect)) {
310 UnicodeString t1, t2;
311 errln(UnicodeString("") +
312 i/3 + ": key(" + IntlTest::prettify(s1, t1)
313 + ").compareTo(key(" + IntlTest::prettify(s2, t2)
314 + ")) got " + result + "; expected " + expect);
315
316 errln((UnicodeString)" " + prettify(k1, t1) + " vs. " + prettify(k2, t2));
317 }
318 }
319 #endif
320 }
321 }
322
sign(int32_t i)323 int8_t CollationThaiTest::sign(int32_t i) {
324 if (i < 0) return -1;
325 if (i > 0) return 1;
326 return 0;
327 }
328
329 /**
330 * Set a UnicodeString corresponding to the given string. Use
331 * UnicodeString and the default converter, unless we see the sequence
332 * "\\u", in which case we interpret the subsequent escape.
333 */
parseChars(UnicodeString & result,const char * chars)334 UnicodeString& CollationThaiTest::parseChars(UnicodeString& result,
335 const char* chars) {
336 return result = CharsToUnicodeString(chars);
337 }
338
339 UCollator *thaiColl = NULL;
340
341 U_CDECL_BEGIN
342 static int U_CALLCONV
StrCmp(const void * p1,const void * p2)343 StrCmp(const void *p1, const void *p2) {
344 return ucol_strcoll(thaiColl, *(UChar **) p1, -1, *(UChar **)p2, -1);
345 }
346 U_CDECL_END
347
348
349 #define LINES 6
350
TestInvalidThai(void)351 void CollationThaiTest::TestInvalidThai(void) {
352 const char *tests[LINES] = {
353 "\\u0E44\\u0E01\\u0E44\\u0E01",
354 "\\u0E44\\u0E01\\u0E01\\u0E44",
355 "\\u0E01\\u0E44\\u0E01\\u0E44",
356 "\\u0E01\\u0E01\\u0E44\\u0E44",
357 "\\u0E44\\u0E44\\u0E01\\u0E01",
358 "\\u0E01\\u0E44\\u0E44\\u0E01",
359 };
360
361 UChar strings[LINES][20];
362
363 UChar *toSort[LINES];
364
365 int32_t i = 0, j = 0, len = 0;
366
367 UErrorCode coll_status = U_ZERO_ERROR;
368 UnicodeString iteratorText;
369
370 thaiColl = ucol_open ("th_TH", &coll_status);
371 if (U_FAILURE(coll_status)) {
372 errln("Error opening Thai collator: %s", u_errorName(coll_status));
373 return;
374 }
375
376 CollationElementIterator* c = ((RuleBasedCollator *)coll)->createCollationElementIterator( iteratorText );
377
378 for(i = 0; i < (int32_t)(sizeof(tests)/sizeof(tests[0])); i++) {
379 len = u_unescape(tests[i], strings[i], 20);
380 strings[i][len] = 0;
381 toSort[i] = strings[i];
382 }
383
384 qsort (toSort, LINES, sizeof (UChar *), StrCmp);
385
386 for (i=0; i < LINES; i++)
387 {
388 logln("%i", i);
389 for (j=i+1; j < LINES; j++) {
390 if (ucol_strcoll (thaiColl, toSort[i], -1, toSort[j], -1) == UCOL_GREATER)
391 {
392 // inconsistency ordering found!
393 errln("Inconsistent ordering between strings %i and %i", i, j);
394 }
395 }
396 iteratorText.setTo(toSort[i]);
397 c->setText(iteratorText, coll_status);
398 backAndForth(*c);
399 }
400
401
402 ucol_close(thaiColl);
403 delete c;
404 }
405
TestReordering(void)406 void CollationThaiTest::TestReordering(void) {
407 // Until UCA 4.1, the collation code swapped Thai/Lao prevowels with the following consonants,
408 // resulting in consonant+prevowel == prevowel+consonant.
409 // From UCA 5.0 on, there are order-reversing contractions for prevowel+consonant.
410 // From UCA 5.0 until UCA 6.1, there was a tertiary difference between
411 // consonant+prevowel and prevowel+consonant.
412 // In UCA 6.2, they compare equal again.
413 // The test was modified to using a collator with strength=secondary,
414 // ignoring possible tertiary differences.
415 const char *tests[] = {
416 "\\u0E41c\\u0301", "=", "\\u0E41\\u0107", // composition
417 "\\u0E41\\U0001D7CE", "<", "\\u0E41\\U0001D7CF", // supplementaries
418 "\\u0E41\\U0001D15F", "=", "\\u0E41\\U0001D158\\U0001D165", // supplementary composition decomps to supplementary
419 "\\u0E41\\U0002F802", "=", "\\u0E41\\u4E41", // supplementary composition decomps to BMP
420 "\\u0E41\\u0301", "=", "\\u0E41\\u0301", // unsafe (just checking backwards iteration)
421 "\\u0E41\\u0301\\u0316", "=", "\\u0E41\\u0316\\u0301",
422
423 "\\u0e24\\u0e41", "=", "\\u0e41\\u0e24", // exiting contraction bug
424 "\\u0e3f\\u0e3f\\u0e24\\u0e41", "=", "\\u0e3f\\u0e3f\\u0e41\\u0e24",
425
426 "abc\\u0E41c\\u0301", "=", "abc\\u0E41\\u0107", // composition
427 "abc\\u0E41\\U0001D000", "<", "abc\\u0E41\\U0001D001", // supplementaries
428 "abc\\u0E41\\U0001D15F", "=", "abc\\u0E41\\U0001D158\\U0001D165", // supplementary composition decomps to supplementary
429 "abc\\u0E41\\U0002F802", "=", "abc\\u0E41\\u4E41", // supplementary composition decomps to BMP
430 "abc\\u0E41\\u0301", "=", "abc\\u0E41\\u0301", // unsafe (just checking backwards iteration)
431 "abc\\u0E41\\u0301\\u0316", "=", "abc\\u0E41\\u0316\\u0301",
432
433 "\\u0E41c\\u0301abc", "=", "\\u0E41\\u0107abc", // composition
434 "\\u0E41\\U0001D000abc", "<", "\\u0E41\\U0001D001abc", // supplementaries
435 "\\u0E41\\U0001D15Fabc", "=", "\\u0E41\\U0001D158\\U0001D165abc", // supplementary composition decomps to supplementary
436 "\\u0E41\\U0002F802abc", "=", "\\u0E41\\u4E41abc", // supplementary composition decomps to BMP
437 "\\u0E41\\u0301abc", "=", "\\u0E41\\u0301abc", // unsafe (just checking backwards iteration)
438 "\\u0E41\\u0301\\u0316abc", "=", "\\u0E41\\u0316\\u0301abc",
439
440 "abc\\u0E41c\\u0301abc", "=", "abc\\u0E41\\u0107abc", // composition
441 "abc\\u0E41\\U0001D000abc", "<", "abc\\u0E41\\U0001D001abc", // supplementaries
442 "abc\\u0E41\\U0001D15Fabc", "=", "abc\\u0E41\\U0001D158\\U0001D165abc", // supplementary composition decomps to supplementary
443 "abc\\u0E41\\U0002F802abc", "=", "abc\\u0E41\\u4E41abc", // supplementary composition decomps to BMP
444 "abc\\u0E41\\u0301abc", "=", "abc\\u0E41\\u0301abc", // unsafe (just checking backwards iteration)
445 "abc\\u0E41\\u0301\\u0316abc", "=", "abc\\u0E41\\u0316\\u0301abc",
446 };
447
448 LocalPointer<Collator> coll2(coll->clone());
449 UErrorCode status = U_ZERO_ERROR;
450 coll2->setAttribute(UCOL_STRENGTH, UCOL_SECONDARY, status);
451 if(U_FAILURE(status)) {
452 errln("Unable to set the Thai collator clone to secondary strength");
453 return;
454 }
455 compareArray(*coll2, tests, sizeof(tests)/sizeof(tests[0]));
456
457 const char *rule = "& c < ab";
458 const char *testcontraction[] = { "\\u0E41ab", ">", "\\u0E41c"}; // After UCA 4.1 Thai are normal so won't break a contraction
459 UnicodeString rules;
460 parseChars(rules, rule);
461 LocalPointer<RuleBasedCollator> rcoll(new RuleBasedCollator(rules, status));
462 if(U_SUCCESS(status)) {
463 compareArray(*rcoll, testcontraction, 3);
464 } else {
465 errln("Couldn't instantiate collator from rules");
466 }
467
468 }
469
470
471 #endif /* #if !UCONFIG_NO_COLLATION */
472