1 /********************************************************************
2 * COPYRIGHT:
3 * Copyright (c) 1997-2013, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
6 /********************************************************************************
7 *
8 * File CBIAPTS.C
9 *
10 * Modification History:
11 * Name Description
12 * Madhu Katragadda Creation
13 *********************************************************************************/
14 /*C API TEST FOR BREAKITERATOR */
15 /**
16 * This is an API test. It doesn't test very many cases, and doesn't
17 * try to test the full functionality. It just calls each function in the class and
18 * verifies that it works on a basic level.
19 **/
20
21 #include "unicode/utypes.h"
22
23 #if !UCONFIG_NO_BREAK_ITERATION
24
25 #include <stdlib.h>
26 #include <string.h>
27 #include "unicode/uloc.h"
28 #include "unicode/ubrk.h"
29 #include "unicode/ustring.h"
30 #include "unicode/ucnv.h"
31 #include "unicode/utext.h"
32 #include "cintltst.h"
33 #include "cbiapts.h"
34
35 #define TEST_ASSERT_SUCCESS(status) {if (U_FAILURE(status)) { \
36 log_data_err("Failure at file %s, line %d, error = %s (Are you missing data?)\n", __FILE__, __LINE__, u_errorName(status));}}
37
38 #define TEST_ASSERT(expr) {if ((expr)==FALSE) { \
39 log_data_err("Test Failure at file %s, line %d (Are you missing data?)\n", __FILE__, __LINE__);}}
40
41 #if !UCONFIG_NO_FILE_IO
42 static void TestBreakIteratorSafeClone(void);
43 #endif
44 static void TestBreakIteratorRules(void);
45 static void TestBreakIteratorRuleError(void);
46 static void TestBreakIteratorStatusVec(void);
47 static void TestBreakIteratorUText(void);
48 static void TestBreakIteratorTailoring(void);
49 static void TestBreakIteratorRefresh(void);
50
51 void addBrkIterAPITest(TestNode** root);
52
addBrkIterAPITest(TestNode ** root)53 void addBrkIterAPITest(TestNode** root)
54 {
55 #if !UCONFIG_NO_FILE_IO
56 addTest(root, &TestBreakIteratorCAPI, "tstxtbd/cbiapts/TestBreakIteratorCAPI");
57 addTest(root, &TestBreakIteratorSafeClone, "tstxtbd/cbiapts/TestBreakIteratorSafeClone");
58 addTest(root, &TestBreakIteratorUText, "tstxtbd/cbiapts/TestBreakIteratorUText");
59 #endif
60 addTest(root, &TestBreakIteratorRules, "tstxtbd/cbiapts/TestBreakIteratorRules");
61 addTest(root, &TestBreakIteratorRuleError, "tstxtbd/cbiapts/TestBreakIteratorRuleError");
62 addTest(root, &TestBreakIteratorStatusVec, "tstxtbd/cbiapts/TestBreakIteratorStatusVec");
63 addTest(root, &TestBreakIteratorTailoring, "tstxtbd/cbiapts/TestBreakIteratorTailoring");
64 addTest(root, &TestBreakIteratorRefresh, "tstxtbd/cbiapts/TestBreakIteratorRefresh");
65 }
66
67 #define CLONETEST_ITERATOR_COUNT 2
68
69 /*
70 * Utility function for converting char * to UChar * strings, to
71 * simplify the test code. Converted strings are put in heap allocated
72 * storage. A hook (probably a local in the caller's code) allows all
73 * strings converted with that hook to be freed with a single call.
74 */
75 typedef struct StringStruct {
76 struct StringStruct *link;
77 UChar str[1];
78 } StringStruct;
79
80
toUChar(const char * src,void ** freeHook)81 static UChar* toUChar(const char *src, void **freeHook) {
82 /* Structure of the memory that we allocate on the heap */
83
84 int32_t numUChars;
85 int32_t destSize;
86 UChar stackBuf[2000 + sizeof(void *)/sizeof(UChar)];
87 StringStruct *dest;
88 UConverter *cnv;
89
90 UErrorCode status = U_ZERO_ERROR;
91 if (src == NULL) {
92 return NULL;
93 };
94
95 cnv = ucnv_open(NULL, &status);
96 if(U_FAILURE(status) || cnv == NULL) {
97 return NULL;
98 }
99 ucnv_reset(cnv);
100 numUChars = ucnv_toUChars(cnv,
101 stackBuf,
102 2000,
103 src, -1,
104 &status);
105
106 destSize = (numUChars+1) * sizeof(UChar) + sizeof(struct StringStruct);
107 dest = (StringStruct *)malloc(destSize);
108 if (dest != NULL) {
109 if (status == U_BUFFER_OVERFLOW_ERROR || status == U_STRING_NOT_TERMINATED_WARNING) {
110 ucnv_toUChars(cnv, dest->str, numUChars+1, src, -1, &status);
111 } else if (status == U_ZERO_ERROR) {
112 u_strcpy(dest->str, stackBuf);
113 } else {
114 free(dest);
115 dest = NULL;
116 }
117 }
118
119 ucnv_reset(cnv); /* be good citizens */
120 ucnv_close(cnv);
121 if (dest == NULL) {
122 return NULL;
123 }
124
125 dest->link = (StringStruct*)(*freeHook);
126 *freeHook = dest;
127 return dest->str;
128 }
129
freeToUCharStrings(void ** hook)130 static void freeToUCharStrings(void **hook) {
131 StringStruct *s = *(StringStruct **)hook;
132 while (s != NULL) {
133 StringStruct *next = s->link;
134 free(s);
135 s = next;
136 }
137 }
138
139
140 #if !UCONFIG_NO_FILE_IO
TestBreakIteratorCAPI()141 static void TestBreakIteratorCAPI()
142 {
143 UErrorCode status = U_ZERO_ERROR;
144 UBreakIterator *word, *sentence, *line, *character, *b, *bogus;
145 int32_t start,pos,end,to;
146 int32_t i;
147 int32_t count = 0;
148
149 UChar text[50];
150
151 /* Note: the adjacent "" are concatenating strings, not adding a \" to the
152 string, which is probably what whoever wrote this intended. Don't fix,
153 because it would throw off the hard coded break positions in the following
154 tests. */
155 u_uastrcpy(text, "He's from Africa. ""Mr. Livingston, I presume?"" Yeah");
156
157
158 /*test ubrk_open()*/
159 log_verbose("\nTesting BreakIterator open functions\n");
160
161 /* Use french for fun */
162 word = ubrk_open(UBRK_WORD, "en_US", text, u_strlen(text), &status);
163 if(status == U_FILE_ACCESS_ERROR) {
164 log_data_err("Check your data - it doesn't seem to be around\n");
165 return;
166 } else if(U_FAILURE(status)){
167 log_err_status(status, "FAIL: Error in ubrk_open() for word breakiterator: %s\n", myErrorName(status));
168 }
169 else{
170 log_verbose("PASS: Successfully opened word breakiterator\n");
171 }
172
173 sentence = ubrk_open(UBRK_SENTENCE, "en_US", text, u_strlen(text), &status);
174 if(U_FAILURE(status)){
175 log_err_status(status, "FAIL: Error in ubrk_open() for sentence breakiterator: %s\n", myErrorName(status));
176 return;
177 }
178 else{
179 log_verbose("PASS: Successfully opened sentence breakiterator\n");
180 }
181
182 line = ubrk_open(UBRK_LINE, "en_US", text, u_strlen(text), &status);
183 if(U_FAILURE(status)){
184 log_err("FAIL: Error in ubrk_open() for line breakiterator: %s\n", myErrorName(status));
185 return;
186 }
187 else{
188 log_verbose("PASS: Successfully opened line breakiterator\n");
189 }
190
191 character = ubrk_open(UBRK_CHARACTER, "en_US", text, u_strlen(text), &status);
192 if(U_FAILURE(status)){
193 log_err("FAIL: Error in ubrk_open() for character breakiterator: %s\n", myErrorName(status));
194 return;
195 }
196 else{
197 log_verbose("PASS: Successfully opened character breakiterator\n");
198 }
199 /*trying to open an illegal iterator*/
200 bogus = ubrk_open((UBreakIteratorType)5, "en_US", text, u_strlen(text), &status);
201 if(bogus != NULL) {
202 log_err("FAIL: expected NULL from opening an invalid break iterator.\n");
203 }
204 if(U_SUCCESS(status)){
205 log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n");
206 }
207 if(U_FAILURE(status)){
208 if(status != U_ILLEGAL_ARGUMENT_ERROR){
209 log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n Got %s\n", myErrorName(status));
210 }
211 }
212 status=U_ZERO_ERROR;
213
214
215 /* ======= Test ubrk_countAvialable() and ubrk_getAvialable() */
216
217 log_verbose("\nTesting ubrk_countAvailable() and ubrk_getAvailable()\n");
218 count=ubrk_countAvailable();
219 /* use something sensible w/o hardcoding the count */
220 if(count < 0){
221 log_err("FAIL: Error in ubrk_countAvialable() returned %d\n", count);
222 }
223 else{
224 log_verbose("PASS: ubrk_countAvialable() successful returned %d\n", count);
225 }
226 for(i=0;i<count;i++)
227 {
228 log_verbose("%s\n", ubrk_getAvailable(i));
229 if (ubrk_getAvailable(i) == 0)
230 log_err("No locale for which breakiterator is applicable\n");
231 else
232 log_verbose("A locale %s for which breakiterator is applicable\n",ubrk_getAvailable(i));
233 }
234
235 /*========Test ubrk_first(), ubrk_last()...... and other functions*/
236
237 log_verbose("\nTesting the functions for word\n");
238 start = ubrk_first(word);
239 if(start!=0)
240 log_err("error ubrk_start(word) did not return 0\n");
241 log_verbose("first (word = %d\n", (int32_t)start);
242 pos=ubrk_next(word);
243 if(pos!=4)
244 log_err("error ubrk_next(word) did not return 4\n");
245 log_verbose("next (word = %d\n", (int32_t)pos);
246 pos=ubrk_following(word, 4);
247 if(pos!=5)
248 log_err("error ubrl_following(word,4) did not return 6\n");
249 log_verbose("next (word = %d\n", (int32_t)pos);
250 end=ubrk_last(word);
251 if(end!=49)
252 log_err("error ubrk_last(word) did not return 49\n");
253 log_verbose("last (word = %d\n", (int32_t)end);
254
255 pos=ubrk_previous(word);
256 log_verbose("%d %d\n", end, pos);
257
258 pos=ubrk_previous(word);
259 log_verbose("%d \n", pos);
260
261 if (ubrk_isBoundary(word, 2) != FALSE) {
262 log_err("error ubrk_isBoundary(word, 2) did not return FALSE\n");
263 }
264 pos=ubrk_current(word);
265 if (pos != 4) {
266 log_err("error ubrk_current() != 4 after ubrk_isBoundary(word, 2)\n");
267 }
268 if (ubrk_isBoundary(word, 4) != TRUE) {
269 log_err("error ubrk_isBoundary(word, 4) did not return TRUE\n");
270 }
271
272
273
274 log_verbose("\nTesting the functions for character\n");
275 ubrk_first(character);
276 pos = ubrk_following(character, 5);
277 if(pos!=6)
278 log_err("error ubrk_following(character,5) did not return 6\n");
279 log_verbose("Following (character,5) = %d\n", (int32_t)pos);
280 pos=ubrk_following(character, 18);
281 if(pos!=19)
282 log_err("error ubrk_following(character,18) did not return 19\n");
283 log_verbose("Followingcharacter,18) = %d\n", (int32_t)pos);
284 pos=ubrk_preceding(character, 22);
285 if(pos!=21)
286 log_err("error ubrk_preceding(character,22) did not return 21\n");
287 log_verbose("preceding(character,22) = %d\n", (int32_t)pos);
288
289
290 log_verbose("\nTesting the functions for line\n");
291 pos=ubrk_first(line);
292 if(pos != 0)
293 log_err("error ubrk_first(line) returned %d, expected 0\n", (int32_t)pos);
294 pos = ubrk_next(line);
295 pos=ubrk_following(line, 18);
296 if(pos!=22)
297 log_err("error ubrk_following(line) did not return 22\n");
298 log_verbose("following (line) = %d\n", (int32_t)pos);
299
300
301 log_verbose("\nTesting the functions for sentence\n");
302 ubrk_first(sentence);
303 pos = ubrk_current(sentence);
304 log_verbose("Current(sentence) = %d\n", (int32_t)pos);
305 pos = ubrk_last(sentence);
306 if(pos!=49)
307 log_err("error ubrk_last for sentence did not return 49\n");
308 log_verbose("Last (sentence) = %d\n", (int32_t)pos);
309 ubrk_first(sentence);
310 to = ubrk_following( sentence, 0 );
311 if (to == 0) log_err("ubrk_following returned 0\n");
312 to = ubrk_preceding( sentence, to );
313 if (to != 0) log_err("ubrk_preceding didn't return 0\n");
314 if (ubrk_first(sentence)!=ubrk_current(sentence)) {
315 log_err("error in ubrk_first() or ubrk_current()\n");
316 }
317
318
319 /*---- */
320 /*Testing ubrk_open and ubrk_close()*/
321 log_verbose("\nTesting open and close for us locale\n");
322 b = ubrk_open(UBRK_WORD, "fr_FR", text, u_strlen(text), &status);
323 if (U_FAILURE(status)) {
324 log_err("ubrk_open for word returned NULL: %s\n", myErrorName(status));
325 }
326 ubrk_close(b);
327
328 /* Test setText and setUText */
329 {
330 UChar s1[] = {0x41, 0x42, 0x20, 0};
331 UChar s2[] = {0x41, 0x42, 0x43, 0x44, 0x45, 0};
332 UText *ut = NULL;
333 UBreakIterator *bb;
334 int j;
335
336 log_verbose("\nTesting ubrk_setText() and ubrk_setUText()\n");
337 status = U_ZERO_ERROR;
338 bb = ubrk_open(UBRK_WORD, "en_US", NULL, 0, &status);
339 TEST_ASSERT_SUCCESS(status);
340 ubrk_setText(bb, s1, -1, &status);
341 TEST_ASSERT_SUCCESS(status);
342 ubrk_first(bb);
343 j = ubrk_next(bb);
344 TEST_ASSERT(j == 2);
345 ut = utext_openUChars(ut, s2, -1, &status);
346 ubrk_setUText(bb, ut, &status);
347 TEST_ASSERT_SUCCESS(status);
348 j = ubrk_next(bb);
349 TEST_ASSERT(j == 5);
350
351 ubrk_close(bb);
352 utext_close(ut);
353 }
354
355 ubrk_close(word);
356 ubrk_close(sentence);
357 ubrk_close(line);
358 ubrk_close(character);
359 }
360
TestBreakIteratorSafeClone(void)361 static void TestBreakIteratorSafeClone(void)
362 {
363 UChar text[51]; /* Keep this odd to test for 64-bit memory alignment */
364 /* NOTE: This doesn't reliably force mis-alignment of following items. */
365 uint8_t buffer [CLONETEST_ITERATOR_COUNT] [U_BRK_SAFECLONE_BUFFERSIZE];
366 int32_t bufferSize = U_BRK_SAFECLONE_BUFFERSIZE;
367
368 UBreakIterator * someIterators [CLONETEST_ITERATOR_COUNT];
369 UBreakIterator * someClonedIterators [CLONETEST_ITERATOR_COUNT];
370
371 UBreakIterator * brk;
372 UErrorCode status = U_ZERO_ERROR;
373 int32_t start,pos;
374 int32_t i;
375
376 /*Testing ubrk_safeClone */
377
378 /* Note: the adjacent "" are concatenating strings, not adding a \" to the
379 string, which is probably what whoever wrote this intended. Don't fix,
380 because it would throw off the hard coded break positions in the following
381 tests. */
382 u_uastrcpy(text, "He's from Africa. ""Mr. Livingston, I presume?"" Yeah");
383
384 /* US & Thai - rule-based & dictionary based */
385 someIterators[0] = ubrk_open(UBRK_WORD, "en_US", text, u_strlen(text), &status);
386 if(!someIterators[0] || U_FAILURE(status)) {
387 log_data_err("Couldn't open en_US word break iterator - %s\n", u_errorName(status));
388 return;
389 }
390
391 someIterators[1] = ubrk_open(UBRK_WORD, "th_TH", text, u_strlen(text), &status);
392 if(!someIterators[1] || U_FAILURE(status)) {
393 log_data_err("Couldn't open th_TH word break iterator - %s\n", u_errorName(status));
394 return;
395 }
396
397 /* test each type of iterator */
398 for (i = 0; i < CLONETEST_ITERATOR_COUNT; i++)
399 {
400
401 /* Check the various error & informational states */
402
403 /* Null status - just returns NULL */
404 if (NULL != ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, NULL))
405 {
406 log_err("FAIL: Cloned Iterator failed to deal correctly with null status\n");
407 }
408 /* error status - should return 0 & keep error the same */
409 status = U_MEMORY_ALLOCATION_ERROR;
410 if (NULL != ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, &status) || status != U_MEMORY_ALLOCATION_ERROR)
411 {
412 log_err("FAIL: Cloned Iterator failed to deal correctly with incoming error status\n");
413 }
414 status = U_ZERO_ERROR;
415
416 /* Null buffer size pointer is ok */
417 if (NULL == (brk = ubrk_safeClone(someIterators[i], buffer[i], NULL, &status)) || U_FAILURE(status))
418 {
419 log_err("FAIL: Cloned Iterator failed to deal correctly with null bufferSize pointer\n");
420 }
421 ubrk_close(brk);
422 status = U_ZERO_ERROR;
423
424 /* buffer size pointer is 0 - fill in pbufferSize with a size */
425 bufferSize = 0;
426 if (NULL != ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, &status) ||
427 U_FAILURE(status) || bufferSize <= 0)
428 {
429 log_err("FAIL: Cloned Iterator failed a sizing request ('preflighting')\n");
430 }
431 /* Verify our define is large enough */
432 if (U_BRK_SAFECLONE_BUFFERSIZE < bufferSize)
433 {
434 log_err("FAIL: Pre-calculated buffer size is too small - %d but needed %d\n", U_BRK_SAFECLONE_BUFFERSIZE, bufferSize);
435 }
436 /* Verify we can use this run-time calculated size */
437 if (NULL == (brk = ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, &status)) || U_FAILURE(status))
438 {
439 log_err("FAIL: Iterator can't be cloned with run-time size\n");
440 }
441 if (brk)
442 ubrk_close(brk);
443 /* size one byte too small - should allocate & let us know */
444 if (bufferSize > 1) {
445 --bufferSize;
446 }
447 if (NULL == (brk = ubrk_safeClone(someIterators[i], NULL, &bufferSize, &status)) || status != U_SAFECLONE_ALLOCATED_WARNING)
448 {
449 log_err("FAIL: Cloned Iterator failed to deal correctly with too-small buffer size\n");
450 }
451 if (brk)
452 ubrk_close(brk);
453 status = U_ZERO_ERROR;
454 bufferSize = U_BRK_SAFECLONE_BUFFERSIZE;
455
456 /* Null buffer pointer - return Iterator & set error to U_SAFECLONE_ALLOCATED_ERROR */
457 if (NULL == (brk = ubrk_safeClone(someIterators[i], NULL, &bufferSize, &status)) || status != U_SAFECLONE_ALLOCATED_WARNING)
458 {
459 log_err("FAIL: Cloned Iterator failed to deal correctly with null buffer pointer\n");
460 }
461 if (brk)
462 ubrk_close(brk);
463 status = U_ZERO_ERROR;
464
465 /* Mis-aligned buffer pointer. */
466 {
467 char stackBuf[U_BRK_SAFECLONE_BUFFERSIZE+sizeof(void *)];
468
469 brk = ubrk_safeClone(someIterators[i], &stackBuf[1], &bufferSize, &status);
470 if (U_FAILURE(status) || brk == NULL) {
471 log_err("FAIL: Cloned Iterator failed with misaligned buffer pointer\n");
472 }
473 if (status == U_SAFECLONE_ALLOCATED_WARNING) {
474 log_verbose("Cloned Iterator allocated when using a mis-aligned buffer.\n");
475 }
476 if (brk)
477 ubrk_close(brk);
478 }
479
480
481 /* Null Iterator - return NULL & set U_ILLEGAL_ARGUMENT_ERROR */
482 if (NULL != ubrk_safeClone(NULL, buffer[i], &bufferSize, &status) || status != U_ILLEGAL_ARGUMENT_ERROR)
483 {
484 log_err("FAIL: Cloned Iterator failed to deal correctly with null Iterator pointer\n");
485 }
486 status = U_ZERO_ERROR;
487
488 /* Do these cloned Iterators work at all - make a first & next call */
489 bufferSize = U_BRK_SAFECLONE_BUFFERSIZE;
490 someClonedIterators[i] = ubrk_safeClone(someIterators[i], buffer[i], &bufferSize, &status);
491
492 start = ubrk_first(someClonedIterators[i]);
493 if(start!=0)
494 log_err("error ubrk_start(clone) did not return 0\n");
495 pos=ubrk_next(someClonedIterators[i]);
496 if(pos!=4)
497 log_err("error ubrk_next(clone) did not return 4\n");
498
499 ubrk_close(someClonedIterators[i]);
500 ubrk_close(someIterators[i]);
501 }
502 }
503 #endif
504
505
506 /*
507 // Open a break iterator from char * rules. Take care of conversion
508 // of the rules and error checking.
509 */
testOpenRules(char * rules)510 static UBreakIterator * testOpenRules(char *rules) {
511 UErrorCode status = U_ZERO_ERROR;
512 UChar *ruleSourceU = NULL;
513 void *strCleanUp = NULL;
514 UParseError parseErr;
515 UBreakIterator *bi;
516
517 ruleSourceU = toUChar(rules, &strCleanUp);
518
519 bi = ubrk_openRules(ruleSourceU, -1, /* The rules */
520 NULL, -1, /* The text to be iterated over. */
521 &parseErr, &status);
522
523 if (U_FAILURE(status)) {
524 log_data_err("FAIL: ubrk_openRules: ICU Error \"%s\" (Are you missing data?)\n", u_errorName(status));
525 bi = 0;
526 };
527 freeToUCharStrings(&strCleanUp);
528 return bi;
529
530 }
531
532 /*
533 * TestBreakIteratorRules - Verify that a break iterator can be created from
534 * a set of source rules.
535 */
TestBreakIteratorRules()536 static void TestBreakIteratorRules() {
537 /* Rules will keep together any run of letters not including 'a', OR
538 * keep together 'abc', but only when followed by 'def', OTHERWISE
539 * just return one char at a time.
540 */
541 char rules[] = "abc{666}/def;\n [\\p{L} - [a]]* {2}; . {1};";
542 /* 0123456789012345678 */
543 char data[] = "abcdex abcdefgh-def"; /* the test data string */
544 char breaks[] = "** ** * ** *"; /* * the expected break positions */
545 char tags[] = "01 21 6 21 2"; /* expected tag values at break positions */
546 int32_t tagMap[] = {0, 1, 2, 3, 4, 5, 666};
547
548 UChar *uData;
549 void *freeHook = NULL;
550 UErrorCode status = U_ZERO_ERROR;
551 int32_t pos;
552 int i;
553
554 UBreakIterator *bi = testOpenRules(rules);
555 if (bi == NULL) {return;}
556 uData = toUChar(data, &freeHook);
557 ubrk_setText(bi, uData, -1, &status);
558
559 pos = ubrk_first(bi);
560 for (i=0; i<sizeof(breaks); i++) {
561 if (pos == i && breaks[i] != '*') {
562 log_err("FAIL: unexpected break at position %d found\n", pos);
563 break;
564 }
565 if (pos != i && breaks[i] == '*') {
566 log_err("FAIL: expected break at position %d not found.\n", i);
567 break;
568 }
569 if (pos == i) {
570 int32_t tag, expectedTag;
571 tag = ubrk_getRuleStatus(bi);
572 expectedTag = tagMap[tags[i]&0xf];
573 if (tag != expectedTag) {
574 log_err("FAIL: incorrect tag value. Position = %d; expected tag %d, got %d",
575 pos, expectedTag, tag);
576 break;
577 }
578 pos = ubrk_next(bi);
579 }
580 }
581
582 freeToUCharStrings(&freeHook);
583 ubrk_close(bi);
584 }
585
TestBreakIteratorRuleError()586 static void TestBreakIteratorRuleError() {
587 /*
588 * TestBreakIteratorRuleError - Try to create a BI from rules with syntax errors,
589 * check that the error is reported correctly.
590 */
591 char rules[] = " # This is a rule comment on line 1\n"
592 "[:L:]; # this rule is OK.\n"
593 "abcdefg); # Error, mismatched parens\n";
594 UChar *uRules;
595 void *freeHook = NULL;
596 UErrorCode status = U_ZERO_ERROR;
597 UParseError parseErr;
598 UBreakIterator *bi;
599
600 uRules = toUChar(rules, &freeHook);
601 bi = ubrk_openRules(uRules, -1, /* The rules */
602 NULL, -1, /* The text to be iterated over. */
603 &parseErr, &status);
604 if (U_SUCCESS(status)) {
605 log_err("FAIL: construction of break iterator succeeded when it should have failed.\n");
606 ubrk_close(bi);
607 } else {
608 if (parseErr.line != 3 || parseErr.offset != 8) {
609 log_data_err("FAIL: incorrect error position reported. Got line %d, char %d, expected line 3, char 7 (Are you missing data?)\n",
610 parseErr.line, parseErr.offset);
611 }
612 }
613 freeToUCharStrings(&freeHook);
614 }
615
616
617 /*
618 * TestsBreakIteratorStatusVals() Test the ubrk_getRuleStatusVec() funciton
619 */
TestBreakIteratorStatusVec()620 static void TestBreakIteratorStatusVec() {
621 #define RULE_STRING_LENGTH 200
622 UChar rules[RULE_STRING_LENGTH];
623
624 #define TEST_STRING_LENGTH 25
625 UChar testString[TEST_STRING_LENGTH];
626 UBreakIterator *bi = NULL;
627 int32_t pos = 0;
628 int32_t vals[10];
629 int32_t numVals;
630 UErrorCode status = U_ZERO_ERROR;
631
632 u_uastrncpy(rules, "[A-N]{100}; \n"
633 "[a-w]{200}; \n"
634 "[\\p{L}]{300}; \n"
635 "[\\p{N}]{400}; \n"
636 "[0-5]{500}; \n"
637 "!.*;\n", RULE_STRING_LENGTH);
638 u_uastrncpy(testString, "ABC", TEST_STRING_LENGTH);
639
640
641 bi = ubrk_openRules(rules, -1, testString, -1, NULL, &status);
642 TEST_ASSERT_SUCCESS(status);
643 TEST_ASSERT(bi != NULL);
644
645 /* The TEST_ASSERT above should change too... */
646 if (bi != NULL) {
647 pos = ubrk_next(bi);
648 TEST_ASSERT(pos == 1);
649
650 memset(vals, -1, sizeof(vals));
651 numVals = ubrk_getRuleStatusVec(bi, vals, 10, &status);
652 TEST_ASSERT_SUCCESS(status);
653 TEST_ASSERT(numVals == 2);
654 TEST_ASSERT(vals[0] == 100);
655 TEST_ASSERT(vals[1] == 300);
656 TEST_ASSERT(vals[2] == -1);
657
658 numVals = ubrk_getRuleStatusVec(bi, vals, 0, &status);
659 TEST_ASSERT(status == U_BUFFER_OVERFLOW_ERROR);
660 TEST_ASSERT(numVals == 2);
661 }
662
663 ubrk_close(bi);
664 }
665
666
667 /*
668 * static void TestBreakIteratorUText(void);
669 *
670 * Test that ubrk_setUText() is present and works for a simple case.
671 */
TestBreakIteratorUText(void)672 static void TestBreakIteratorUText(void) {
673 const char *UTF8Str = "\x41\xc3\x85\x5A\x20\x41\x52\x69\x6E\x67"; /* c3 85 is utf-8 for A with a ring on top */
674 /* 0 1 2 34567890 */
675
676 UErrorCode status = U_ZERO_ERROR;
677 UBreakIterator *bi = NULL;
678 int32_t pos = 0;
679
680
681 UText *ut = utext_openUTF8(NULL, UTF8Str, -1, &status);
682 TEST_ASSERT_SUCCESS(status);
683
684 bi = ubrk_open(UBRK_WORD, "en_US", NULL, 0, &status);
685 if (U_FAILURE(status)) {
686 log_err_status(status, "Failure at file %s, line %d, error = %s\n", __FILE__, __LINE__, u_errorName(status));
687 return;
688 }
689
690 ubrk_setUText(bi, ut, &status);
691 if (U_FAILURE(status)) {
692 log_err("Failure at file %s, line %d, error = %s\n", __FILE__, __LINE__, u_errorName(status));
693 return;
694 }
695
696 pos = ubrk_first(bi);
697 TEST_ASSERT(pos == 0);
698
699 pos = ubrk_next(bi);
700 TEST_ASSERT(pos == 4);
701
702 pos = ubrk_next(bi);
703 TEST_ASSERT(pos == 5);
704
705 pos = ubrk_next(bi);
706 TEST_ASSERT(pos == 10);
707
708 pos = ubrk_next(bi);
709 TEST_ASSERT(pos == UBRK_DONE);
710 ubrk_close(bi);
711 utext_close(ut);
712 }
713
714 /*
715 * static void TestBreakIteratorTailoring(void);
716 *
717 * Test break iterator tailorings from CLDR data.
718 */
719
720 /* Thai/Lao grapheme break tailoring */
721 static const UChar thTest[] = { 0x0020, 0x0E40, 0x0E01, 0x0020,
722 0x0E01, 0x0E30, 0x0020, 0x0E01, 0x0E33, 0x0020, 0 };
723 /*in Unicode 6.1 en should behave just like th for this*/
724 /*static const int32_t thTestOffs_enFwd[] = { 1, 3, 4, 6, 7, 9, 10 };*/
725 static const int32_t thTestOffs_thFwd[] = { 1, 2, 3, 4, 5, 6, 7, 9, 10 };
726 /*static const int32_t thTestOffs_enRev[] = { 9, 7, 6, 4, 3, 1, 0 };*/
727 static const int32_t thTestOffs_thRev[] = { 9, 7, 6, 5, 4, 3, 2, 1, 0 };
728
729 /* Hebrew line break tailoring, for cldrbug 3028 */
730 static const UChar heTest[] = { 0x0020, 0x002D, 0x0031, 0x0032, 0x0020,
731 0x0061, 0x002D, 0x006B, 0x0020,
732 0x0061, 0x0300, 0x2010, 0x006B, 0x0020,
733 0x05DE, 0x05D4, 0x002D, 0x0069, 0x0020,
734 0x05D1, 0x05BC, 0x2010, 0x0047, 0x0020, 0 };
735 /*in Unicode 6.1 en should behave just like he for this*/
736 /*static const int32_t heTestOffs_enFwd[] = { 1, 5, 7, 9, 12, 14, 17, 19, 22, 24 };*/
737 static const int32_t heTestOffs_heFwd[] = { 1, 5, 7, 9, 12, 14, 19, 24 };
738 /*static const int32_t heTestOffs_enRev[] = { 22, 19, 17, 14, 12, 9, 7, 5, 1, 0 };*/
739 static const int32_t heTestOffs_heRev[] = { 19, 14, 12, 9, 7, 5, 1, 0 };
740
741 /* Finnish line break tailoring, for cldrbug 3029 */
742 static const UChar fiTest[] = { /* 00 */ 0x0020, 0x002D, 0x0031, 0x0032, 0x0020,
743 /* 05 */ 0x0061, 0x002D, 0x006B, 0x0020,
744 /* 09 */ 0x0061, 0x0300, 0x2010, 0x006B, 0x0020,
745 /* 14 */ 0x0061, 0x0020, 0x002D, 0x006B, 0x0020,
746 /* 19 */ 0x0061, 0x0300, 0x0020, 0x2010, 0x006B, 0x0020, 0 };
747 static const int32_t fiTestOffs_enFwd[] = { 1, 5, 7, 9, 12, 14, 16, 17, 19, 22, 23, 25 };
748 static const int32_t fiTestOffs_fiFwd[] = { 1, 5, 7, 9, 12, 14, 16, 19, 22, 25 };
749 static const int32_t fiTestOffs_enRev[] = { 23, 22, 19, 17, 16, 14, 12, 9, 7, 5, 1, 0 };
750 static const int32_t fiTestOffs_fiRev[] = { 22, 19, 16, 14, 12, 9, 7, 5, 1, 0 };
751
752 /* Khmer dictionary-based work break, for ICU ticket #8329 */
753 static const UChar kmTest[] = { /* 00 */ 0x179F, 0x17BC, 0x1798, 0x1785, 0x17C6, 0x178E, 0x17B6, 0x1799, 0x1796, 0x17C1,
754 /* 10 */ 0x179B, 0x1794, 0x1793, 0x17D2, 0x178F, 0x17B7, 0x1785, 0x178A, 0x17BE, 0x1798,
755 /* 20 */ 0x17D2, 0x1794, 0x17B8, 0x17A2, 0x1792, 0x17B7, 0x179F, 0x17D2, 0x178B, 0x17B6,
756 /* 30 */ 0x1793, 0x17A2, 0x179A, 0x1796, 0x17D2, 0x179A, 0x17C7, 0x1782, 0x17BB, 0x178E,
757 /* 40 */ 0x178A, 0x179B, 0x17CB, 0x1796, 0x17D2, 0x179A, 0x17C7, 0x17A2, 0x1784, 0x17D2,
758 /* 50 */ 0x1782, 0 };
759 static const int32_t kmTestOffs_kmFwd[] = { 3, /*8,*/ 11, 17, 23, 31, /*33,*/ 40, 43, 51 }; /* TODO: Investigate failure to break at offset 8 */
760 static const int32_t kmTestOffs_kmRev[] = { 43, 40, /*33,*/ 31, 23, 17, 11, /*8,*/ 3, 0 };
761
762 typedef struct {
763 const char * locale;
764 UBreakIteratorType type;
765 const UChar * test;
766 const int32_t * offsFwd;
767 const int32_t * offsRev;
768 int32_t numOffsets;
769 } RBBITailoringTest;
770
771 static const RBBITailoringTest tailoringTests[] = {
772 { "en", UBRK_CHARACTER, thTest, thTestOffs_thFwd, thTestOffs_thRev, sizeof(thTestOffs_thFwd)/sizeof(thTestOffs_thFwd[0]) },
773 { "en_US_POSIX", UBRK_CHARACTER, thTest, thTestOffs_thFwd, thTestOffs_thRev, sizeof(thTestOffs_thFwd)/sizeof(thTestOffs_thFwd[0]) },
774 { "en", UBRK_LINE, heTest, heTestOffs_heFwd, heTestOffs_heRev, sizeof(heTestOffs_heFwd)/sizeof(heTestOffs_heFwd[0]) },
775 { "he", UBRK_LINE, heTest, heTestOffs_heFwd, heTestOffs_heRev, sizeof(heTestOffs_heFwd)/sizeof(heTestOffs_heFwd[0]) },
776 { "en", UBRK_LINE, fiTest, fiTestOffs_enFwd, fiTestOffs_enRev, sizeof(fiTestOffs_enFwd)/sizeof(fiTestOffs_enFwd[0]) },
777 { "fi", UBRK_LINE, fiTest, fiTestOffs_fiFwd, fiTestOffs_fiRev, sizeof(fiTestOffs_fiFwd)/sizeof(fiTestOffs_fiFwd[0]) },
778 { "km", UBRK_WORD, kmTest, kmTestOffs_kmFwd, kmTestOffs_kmRev, sizeof(kmTestOffs_kmFwd)/sizeof(kmTestOffs_kmFwd[0]) },
779 { NULL, 0, NULL, NULL, NULL, 0 },
780 };
781
TestBreakIteratorTailoring(void)782 static void TestBreakIteratorTailoring(void) {
783 const RBBITailoringTest * testPtr;
784 for (testPtr = tailoringTests; testPtr->locale != NULL; ++testPtr) {
785 UErrorCode status = U_ZERO_ERROR;
786 UBreakIterator* ubrkiter = ubrk_open(testPtr->type, testPtr->locale, testPtr->test, -1, &status);
787 if ( U_SUCCESS(status) ) {
788 int32_t offset, offsindx;
789 UBool foundError;
790
791 foundError = FALSE;
792 for (offsindx = 0; (offset = ubrk_next(ubrkiter)) != UBRK_DONE; ++offsindx) {
793 if (!foundError && offsindx >= testPtr->numOffsets) {
794 log_err("FAIL: locale %s, break type %d, ubrk_next expected UBRK_DONE, got %d\n",
795 testPtr->locale, testPtr->type, offset);
796 foundError = TRUE;
797 } else if (!foundError && offset != testPtr->offsFwd[offsindx]) {
798 log_err("FAIL: locale %s, break type %d, ubrk_next expected %d, got %d\n",
799 testPtr->locale, testPtr->type, testPtr->offsFwd[offsindx], offset);
800 foundError = TRUE;
801 }
802 }
803 if (!foundError && offsindx < testPtr->numOffsets) {
804 log_err("FAIL: locale %s, break type %d, ubrk_next expected %d, got UBRK_DONE\n",
805 testPtr->locale, testPtr->type, testPtr->offsFwd[offsindx]);
806 }
807
808 foundError = FALSE;
809 for (offsindx = 0; (offset = ubrk_previous(ubrkiter)) != UBRK_DONE; ++offsindx) {
810 if (!foundError && offsindx >= testPtr->numOffsets) {
811 log_err("FAIL: locale %s, break type %d, ubrk_previous expected UBRK_DONE, got %d\n",
812 testPtr->locale, testPtr->type, offset);
813 foundError = TRUE;
814 } else if (!foundError && offset != testPtr->offsRev[offsindx]) {
815 log_err("FAIL: locale %s, break type %d, ubrk_previous expected %d, got %d\n",
816 testPtr->locale, testPtr->type, testPtr->offsRev[offsindx], offset);
817 foundError = TRUE;
818 }
819 }
820 if (!foundError && offsindx < testPtr->numOffsets) {
821 log_err("FAIL: locale %s, break type %d, ubrk_previous expected %d, got UBRK_DONE\n",
822 testPtr->locale, testPtr->type, testPtr->offsRev[offsindx]);
823 }
824
825 ubrk_close(ubrkiter);
826 } else {
827 log_err_status(status, "FAIL: locale %s, break type %d, ubrk_open status: %s\n", testPtr->locale, testPtr->type, u_errorName(status));
828 }
829 }
830 }
831
832
TestBreakIteratorRefresh(void)833 static void TestBreakIteratorRefresh(void) {
834 /*
835 * RefreshInput changes out the input of a Break Iterator without
836 * changing anything else in the iterator's state. Used with Java JNI,
837 * when Java moves the underlying string storage. This test
838 * runs a ubrk_next() repeatedly, moving the text in the middle of the sequence.
839 * The right set of boundaries should still be found.
840 */
841 UChar testStr[] = {0x20, 0x41, 0x20, 0x42, 0x20, 0x43, 0x20, 0x44, 0x0}; /* = " A B C D" */
842 UChar movedStr[] = {0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0};
843 UErrorCode status = U_ZERO_ERROR;
844 UBreakIterator *bi;
845 UText ut1 = UTEXT_INITIALIZER;
846 UText ut2 = UTEXT_INITIALIZER;
847
848 bi = ubrk_open(UBRK_LINE, "en_US", NULL, 0, &status);
849 TEST_ASSERT_SUCCESS(status);
850
851 utext_openUChars(&ut1, testStr, -1, &status);
852 TEST_ASSERT_SUCCESS(status);
853 ubrk_setUText(bi, &ut1, &status);
854 TEST_ASSERT_SUCCESS(status);
855
856 if (U_SUCCESS(status)) {
857 /* Line boundaries will occur before each letter in the original string */
858 TEST_ASSERT(1 == ubrk_next(bi));
859 TEST_ASSERT(3 == ubrk_next(bi));
860
861 /* Move the string, kill the original string. */
862 u_strcpy(movedStr, testStr);
863 u_memset(testStr, 0x20, u_strlen(testStr));
864 utext_openUChars(&ut2, movedStr, -1, &status);
865 TEST_ASSERT_SUCCESS(status);
866 ubrk_refreshUText(bi, &ut2, &status);
867 TEST_ASSERT_SUCCESS(status);
868
869 /* Find the following matches, now working in the moved string. */
870 TEST_ASSERT(5 == ubrk_next(bi));
871 TEST_ASSERT(7 == ubrk_next(bi));
872 TEST_ASSERT(8 == ubrk_next(bi));
873 TEST_ASSERT(UBRK_DONE == ubrk_next(bi));
874 TEST_ASSERT_SUCCESS(status);
875
876 utext_close(&ut1);
877 utext_close(&ut2);
878 }
879 ubrk_close(bi);
880 }
881
882 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
883