1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 2003-2013, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: convtest.cpp
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 2003jul15
14 * created by: Markus W. Scherer
15 *
16 * Test file for data-driven conversion tests.
17 */
18
19 #include "unicode/utypes.h"
20
21 #if !UCONFIG_NO_LEGACY_CONVERSION
22 /*
23 * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
24 * is slightly unnecessary - it removes tests for Unicode charsets
25 * like UTF-8 that should work.
26 * However, there is no easy way for the test to detect whether a test case
27 * is for a Unicode charset, so it would be difficult to only exclude those.
28 * Also, regular testing of ICU is done with all modules on, therefore
29 * not testing conversion for a custom configuration like this should be ok.
30 */
31
32 #include "unicode/ucnv.h"
33 #include "unicode/unistr.h"
34 #include "unicode/parsepos.h"
35 #include "unicode/uniset.h"
36 #include "unicode/ustring.h"
37 #include "unicode/ures.h"
38 #include "convtest.h"
39 #include "unicode/tstdtmod.h"
40 #include <string.h>
41 #include <stdlib.h>
42
43 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
44
45 enum {
46 // characters used in test data for callbacks
47 SUB_CB='?',
48 SKIP_CB='0',
49 STOP_CB='.',
50 ESC_CB='&'
51 };
52
ConversionTest()53 ConversionTest::ConversionTest() {
54 UErrorCode errorCode=U_ZERO_ERROR;
55 utf8Cnv=ucnv_open("UTF-8", &errorCode);
56 ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
57 if(U_FAILURE(errorCode)) {
58 errln("unable to open UTF-8 converter");
59 }
60 }
61
~ConversionTest()62 ConversionTest::~ConversionTest() {
63 ucnv_close(utf8Cnv);
64 }
65
66 void
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)67 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
68 if (exec) logln("TestSuite ConversionTest: ");
69 switch (index) {
70 #if !UCONFIG_NO_FILE_IO
71 case 0: name="TestToUnicode"; if (exec) TestToUnicode(); break;
72 case 1: name="TestFromUnicode"; if (exec) TestFromUnicode(); break;
73 case 2: name="TestGetUnicodeSet"; if (exec) TestGetUnicodeSet(); break;
74 #else
75 case 0:
76 case 1:
77 case 2: name="skip"; break;
78 #endif
79 case 3: name="TestGetUnicodeSet2"; if (exec) TestGetUnicodeSet2(); break;
80 default: name=""; break; //needed to end loop
81 }
82 }
83
84 // test data interface ----------------------------------------------------- ***
85
86 void
TestToUnicode()87 ConversionTest::TestToUnicode() {
88 ConversionCase cc;
89 char charset[100], cbopt[4];
90 const char *option;
91 UnicodeString s, unicode;
92 int32_t offsetsLength;
93 UConverterToUCallback callback;
94
95 TestDataModule *dataModule;
96 TestData *testData;
97 const DataMap *testCase;
98 UErrorCode errorCode;
99 int32_t i;
100
101 errorCode=U_ZERO_ERROR;
102 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
103 if(U_SUCCESS(errorCode)) {
104 testData=dataModule->createTestData("toUnicode", errorCode);
105 if(U_SUCCESS(errorCode)) {
106 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
107 if(U_FAILURE(errorCode)) {
108 errln("error retrieving conversion/toUnicode test case %d - %s",
109 i, u_errorName(errorCode));
110 errorCode=U_ZERO_ERROR;
111 continue;
112 }
113
114 cc.caseNr=i;
115
116 s=testCase->getString("charset", errorCode);
117 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
118 cc.charset=charset;
119
120 // BEGIN android-added
121 // To save space, Android does not build full ISO-2022-CN tables.
122 // We skip the TestGetKeywordValuesForLocale for counting available collations.
123 if (strlen(charset) >= 8 &&
124 strncmp(charset+4, "2022-CN", 4) == 0) {
125 continue;
126 }
127 // END android-added
128
129 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
130 unicode=testCase->getString("unicode", errorCode);
131 cc.unicode=unicode.getBuffer();
132 cc.unicodeLength=unicode.length();
133
134 offsetsLength=0;
135 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
136 if(offsetsLength==0) {
137 cc.offsets=NULL;
138 } else if(offsetsLength!=unicode.length()) {
139 errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
140 i, unicode.length(), offsetsLength);
141 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
142 }
143
144 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
145 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
146
147 s=testCase->getString("errorCode", errorCode);
148 if(s==UNICODE_STRING("invalid", 7)) {
149 cc.outErrorCode=U_INVALID_CHAR_FOUND;
150 } else if(s==UNICODE_STRING("illegal", 7)) {
151 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
152 } else if(s==UNICODE_STRING("truncated", 9)) {
153 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
154 } else if(s==UNICODE_STRING("illesc", 6)) {
155 cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
156 } else if(s==UNICODE_STRING("unsuppesc", 9)) {
157 cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
158 } else {
159 cc.outErrorCode=U_ZERO_ERROR;
160 }
161
162 s=testCase->getString("callback", errorCode);
163 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
164 cc.cbopt=cbopt;
165 switch(cbopt[0]) {
166 case SUB_CB:
167 callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
168 break;
169 case SKIP_CB:
170 callback=UCNV_TO_U_CALLBACK_SKIP;
171 break;
172 case STOP_CB:
173 callback=UCNV_TO_U_CALLBACK_STOP;
174 break;
175 case ESC_CB:
176 callback=UCNV_TO_U_CALLBACK_ESCAPE;
177 break;
178 default:
179 callback=NULL;
180 break;
181 }
182 option=callback==NULL ? cbopt : cbopt+1;
183 if(*option==0) {
184 option=NULL;
185 }
186
187 cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
188
189 if(U_FAILURE(errorCode)) {
190 errln("error parsing conversion/toUnicode test case %d - %s",
191 i, u_errorName(errorCode));
192 errorCode=U_ZERO_ERROR;
193 } else {
194 logln("TestToUnicode[%d] %s", i, charset);
195 ToUnicodeCase(cc, callback, option);
196 }
197 }
198 delete testData;
199 }
200 delete dataModule;
201 }
202 else {
203 dataerrln("Could not load test conversion data");
204 }
205 }
206
207 void
TestFromUnicode()208 ConversionTest::TestFromUnicode() {
209 ConversionCase cc;
210 char charset[100], cbopt[4];
211 const char *option;
212 UnicodeString s, unicode, invalidUChars;
213 int32_t offsetsLength, index;
214 UConverterFromUCallback callback;
215
216 TestDataModule *dataModule;
217 TestData *testData;
218 const DataMap *testCase;
219 const UChar *p;
220 UErrorCode errorCode;
221 int32_t i, length;
222
223 errorCode=U_ZERO_ERROR;
224 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
225 if(U_SUCCESS(errorCode)) {
226 testData=dataModule->createTestData("fromUnicode", errorCode);
227 if(U_SUCCESS(errorCode)) {
228 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
229 if(U_FAILURE(errorCode)) {
230 errln("error retrieving conversion/fromUnicode test case %d - %s",
231 i, u_errorName(errorCode));
232 errorCode=U_ZERO_ERROR;
233 continue;
234 }
235
236 cc.caseNr=i;
237
238 s=testCase->getString("charset", errorCode);
239 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
240 cc.charset=charset;
241
242 // BEGIN android-added
243 // To save space, Android does not build full ISO-2022-CN tables.
244 // We skip the TestGetKeywordValuesForLocale for counting available collations.
245 if (strlen(charset) >= 8 &&
246 strncmp(charset+4, "2022-CN", 4) == 0) {
247 continue;
248 }
249 // END android-added
250
251 unicode=testCase->getString("unicode", errorCode);
252 cc.unicode=unicode.getBuffer();
253 cc.unicodeLength=unicode.length();
254 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
255
256 offsetsLength=0;
257 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
258 if(offsetsLength==0) {
259 cc.offsets=NULL;
260 } else if(offsetsLength!=cc.bytesLength) {
261 errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
262 i, cc.bytesLength, offsetsLength);
263 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
264 }
265
266 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
267 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
268
269 s=testCase->getString("errorCode", errorCode);
270 if(s==UNICODE_STRING("invalid", 7)) {
271 cc.outErrorCode=U_INVALID_CHAR_FOUND;
272 } else if(s==UNICODE_STRING("illegal", 7)) {
273 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
274 } else if(s==UNICODE_STRING("truncated", 9)) {
275 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
276 } else {
277 cc.outErrorCode=U_ZERO_ERROR;
278 }
279
280 s=testCase->getString("callback", errorCode);
281 cc.setSub=0; // default: no subchar
282
283 if((index=s.indexOf((UChar)0))>0) {
284 // read NUL-separated subchar first, if any
285 // copy the subchar from Latin-1 characters
286 // start after the NUL
287 p=s.getTerminatedBuffer();
288 length=index+1;
289 p+=length;
290 length=s.length()-length;
291 if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
292 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
293 } else {
294 int32_t j;
295
296 for(j=0; j<length; ++j) {
297 cc.subchar[j]=(char)p[j];
298 }
299 // NUL-terminate the subchar
300 cc.subchar[j]=0;
301 cc.setSub=1;
302 }
303
304 // remove the NUL and subchar from s
305 s.truncate(index);
306 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
307 // read a substitution string, separated by an equal sign
308 p=s.getBuffer()+index+1;
309 length=s.length()-(index+1);
310 if(length<0 || length>=LENGTHOF(cc.subString)) {
311 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
312 } else {
313 u_memcpy(cc.subString, p, length);
314 // NUL-terminate the subString
315 cc.subString[length]=0;
316 cc.setSub=-1;
317 }
318
319 // remove the equal sign and subString from s
320 s.truncate(index);
321 }
322
323 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
324 cc.cbopt=cbopt;
325 switch(cbopt[0]) {
326 case SUB_CB:
327 callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
328 break;
329 case SKIP_CB:
330 callback=UCNV_FROM_U_CALLBACK_SKIP;
331 break;
332 case STOP_CB:
333 callback=UCNV_FROM_U_CALLBACK_STOP;
334 break;
335 case ESC_CB:
336 callback=UCNV_FROM_U_CALLBACK_ESCAPE;
337 break;
338 default:
339 callback=NULL;
340 break;
341 }
342 option=callback==NULL ? cbopt : cbopt+1;
343 if(*option==0) {
344 option=NULL;
345 }
346
347 invalidUChars=testCase->getString("invalidUChars", errorCode);
348 cc.invalidUChars=invalidUChars.getBuffer();
349 cc.invalidLength=invalidUChars.length();
350
351 if(U_FAILURE(errorCode)) {
352 errln("error parsing conversion/fromUnicode test case %d - %s",
353 i, u_errorName(errorCode));
354 errorCode=U_ZERO_ERROR;
355 } else {
356 logln("TestFromUnicode[%d] %s", i, charset);
357 FromUnicodeCase(cc, callback, option);
358 }
359 }
360 delete testData;
361 }
362 delete dataModule;
363 }
364 else {
365 dataerrln("Could not load test conversion data");
366 }
367 }
368
369 static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
370
371 void
TestGetUnicodeSet()372 ConversionTest::TestGetUnicodeSet() {
373 char charset[100];
374 UnicodeString s, map, mapnot;
375 int32_t which;
376
377 ParsePosition pos;
378 UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
379 UnicodeSet *cnvSetPtr = &cnvSet;
380 LocalUConverterPointer cnv;
381
382 TestDataModule *dataModule;
383 TestData *testData;
384 const DataMap *testCase;
385 UErrorCode errorCode;
386 int32_t i;
387
388 errorCode=U_ZERO_ERROR;
389 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
390 if(U_SUCCESS(errorCode)) {
391 testData=dataModule->createTestData("getUnicodeSet", errorCode);
392 if(U_SUCCESS(errorCode)) {
393 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
394 if(U_FAILURE(errorCode)) {
395 errln("error retrieving conversion/getUnicodeSet test case %d - %s",
396 i, u_errorName(errorCode));
397 errorCode=U_ZERO_ERROR;
398 continue;
399 }
400
401 s=testCase->getString("charset", errorCode);
402 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
403
404 // BEGIN android-added
405 // To save space, Android does not build full ISO-2022-CN tables.
406 // We skip the TestGetKeywordValuesForLocale for counting available collations.
407 if (strlen(charset) >= 8 &&
408 strncmp(charset+4, "2022-CN", 4) == 0) {
409 continue;
410 }
411 // END android-added
412
413 map=testCase->getString("map", errorCode);
414 mapnot=testCase->getString("mapnot", errorCode);
415
416 which=testCase->getInt28("which", errorCode);
417
418 if(U_FAILURE(errorCode)) {
419 errln("error parsing conversion/getUnicodeSet test case %d - %s",
420 i, u_errorName(errorCode));
421 errorCode=U_ZERO_ERROR;
422 continue;
423 }
424
425 // test this test case
426 mapSet.clear();
427 mapnotSet.clear();
428
429 pos.setIndex(0);
430 mapSet.applyPattern(map, pos, 0, NULL, errorCode);
431 if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
432 errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
433 " error index %d index %d U+%04x",
434 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
435 errorCode=U_ZERO_ERROR;
436 continue;
437 }
438
439 pos.setIndex(0);
440 mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
441 if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
442 errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
443 " error index %d index %d U+%04x",
444 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
445 errorCode=U_ZERO_ERROR;
446 continue;
447 }
448
449 logln("TestGetUnicodeSet[%d] %s", i, charset);
450
451 cnv.adoptInstead(cnv_open(charset, errorCode));
452 if(U_FAILURE(errorCode)) {
453 errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
454 charset, i, u_errorName(errorCode));
455 errorCode=U_ZERO_ERROR;
456 continue;
457 }
458
459 ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
460
461 if(U_FAILURE(errorCode)) {
462 errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
463 charset, i, u_errorName(errorCode));
464 errorCode=U_ZERO_ERROR;
465 continue;
466 }
467
468 // are there items that must be in cnvSet but are not?
469 (diffSet=mapSet).removeAll(cnvSet);
470 if(!diffSet.isEmpty()) {
471 diffSet.toPattern(s, TRUE);
472 if(s.length()>100) {
473 s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
474 }
475 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
476 charset, i);
477 errln(s);
478 }
479
480 // are there items that must not be in cnvSet but are?
481 (diffSet=mapnotSet).retainAll(cnvSet);
482 if(!diffSet.isEmpty()) {
483 diffSet.toPattern(s, TRUE);
484 if(s.length()>100) {
485 s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
486 }
487 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
488 charset, i);
489 errln(s);
490 }
491 }
492 delete testData;
493 }
494 delete dataModule;
495 }
496 else {
497 dataerrln("Could not load test conversion data");
498 }
499 }
500
501 U_CDECL_BEGIN
502 static void U_CALLCONV
getUnicodeSetCallback(const void * context,UConverterFromUnicodeArgs *,const UChar *,int32_t,UChar32 codePoint,UConverterCallbackReason reason,UErrorCode * pErrorCode)503 getUnicodeSetCallback(const void *context,
504 UConverterFromUnicodeArgs * /*fromUArgs*/,
505 const UChar* /*codeUnits*/,
506 int32_t /*length*/,
507 UChar32 codePoint,
508 UConverterCallbackReason reason,
509 UErrorCode *pErrorCode) {
510 if(reason<=UCNV_IRREGULAR) {
511 ((UnicodeSet *)context)->remove(codePoint); // the converter cannot convert this code point
512 *pErrorCode=U_ZERO_ERROR; // skip
513 } // else ignore the reset, close and clone calls.
514 }
515 U_CDECL_END
516
517 // Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
518 void
TestGetUnicodeSet2()519 ConversionTest::TestGetUnicodeSet2() {
520 // Build a string with all code points.
521 UChar32 cpLimit;
522 int32_t s0Length;
523 if(quick) {
524 cpLimit=s0Length=0x10000; // BMP only
525 } else {
526 cpLimit=0x110000;
527 s0Length=0x10000+0x200000; // BMP + surrogate pairs
528 }
529 UChar *s0=new UChar[s0Length];
530 if(s0==NULL) {
531 return;
532 }
533 UChar *s=s0;
534 UChar32 c;
535 UChar c2;
536 // low BMP
537 for(c=0; c<=0xd7ff; ++c) {
538 *s++=(UChar)c;
539 }
540 // trail surrogates
541 for(c=0xdc00; c<=0xdfff; ++c) {
542 *s++=(UChar)c;
543 }
544 // lead surrogates
545 // (after trails so that there is not even one surrogate pair in between)
546 for(c=0xd800; c<=0xdbff; ++c) {
547 *s++=(UChar)c;
548 }
549 // high BMP
550 for(c=0xe000; c<=0xffff; ++c) {
551 *s++=(UChar)c;
552 }
553 // supplementary code points = surrogate pairs
554 if(cpLimit==0x110000) {
555 for(c=0xd800; c<=0xdbff; ++c) {
556 for(c2=0xdc00; c2<=0xdfff; ++c2) {
557 *s++=(UChar)c;
558 *s++=c2;
559 }
560 }
561 }
562
563 static const char *const cnvNames[]={
564 "UTF-8",
565 "UTF-7",
566 "UTF-16",
567 "US-ASCII",
568 "ISO-8859-1",
569 "windows-1252",
570 "Shift-JIS",
571 "ibm-1390", // EBCDIC_STATEFUL table
572 "ibm-16684", // DBCS-only extension table based on EBCDIC_STATEFUL table
573 "HZ",
574 "ISO-2022-JP",
575 "JIS7",
576 "ISO-2022-CN",
577 "ISO-2022-CN-EXT",
578 "LMBCS"
579 };
580 LocalUConverterPointer cnv;
581 char buffer[1024];
582 int32_t i;
583 for(i=0; i<LENGTHOF(cnvNames); ++i) {
584 UErrorCode errorCode=U_ZERO_ERROR;
585 cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
586 if(U_FAILURE(errorCode)) {
587 errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
588 continue;
589 }
590 UnicodeSet expected;
591 ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
592 if(U_FAILURE(errorCode)) {
593 errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
594 continue;
595 }
596 UConverterUnicodeSet which;
597 for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
598 if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
599 ucnv_setFallback(cnv.getAlias(), TRUE);
600 }
601 expected.add(0, cpLimit-1);
602 s=s0;
603 UBool flush;
604 do {
605 char *t=buffer;
606 flush=(UBool)(s==s0+s0Length);
607 ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
608 if(U_FAILURE(errorCode)) {
609 if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
610 errorCode=U_ZERO_ERROR;
611 continue;
612 } else {
613 break; // unexpected error, should not occur
614 }
615 }
616 } while(!flush);
617 UnicodeSet set;
618 ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
619 if(cpLimit<0x110000) {
620 set.remove(cpLimit, 0x10ffff);
621 }
622 if(which==UCNV_ROUNDTRIP_SET) {
623 // ignore PUA code points because they will be converted even if they
624 // are fallbacks and when other fallbacks are turned off,
625 // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
626 expected.remove(0xe000, 0xf8ff);
627 expected.remove(0xf0000, 0xffffd);
628 expected.remove(0x100000, 0x10fffd);
629 set.remove(0xe000, 0xf8ff);
630 set.remove(0xf0000, 0xffffd);
631 set.remove(0x100000, 0x10fffd);
632 }
633 if(set!=expected) {
634 // First try to see if we have different sets because ucnv_getUnicodeSet()
635 // added strings: The above conversion method does not tell us what strings might be convertible.
636 // Remove strings from the set and compare again.
637 // Unfortunately, there are no good, direct set methods for finding out whether there are strings
638 // in the set, nor for enumerating or removing just them.
639 // Intersect all code points with the set. The intersection will not contain strings.
640 UnicodeSet temp(0, 0x10ffff);
641 temp.retainAll(set);
642 set=temp;
643 }
644 if(set!=expected) {
645 UnicodeSet diffSet;
646 UnicodeString out;
647
648 // are there items that must be in the set but are not?
649 (diffSet=expected).removeAll(set);
650 if(!diffSet.isEmpty()) {
651 diffSet.toPattern(out, TRUE);
652 if(out.length()>100) {
653 out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
654 }
655 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
656 cnvNames[i], which);
657 errln(out);
658 }
659
660 // are there items that must not be in the set but are?
661 (diffSet=set).removeAll(expected);
662 if(!diffSet.isEmpty()) {
663 diffSet.toPattern(out, TRUE);
664 if(out.length()>100) {
665 out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
666 }
667 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
668 cnvNames[i], which);
669 errln(out);
670 }
671 }
672 }
673 }
674
675 delete [] s0;
676 }
677
678 // open testdata or ICU data converter ------------------------------------- ***
679
680 UConverter *
cnv_open(const char * name,UErrorCode & errorCode)681 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
682 if(name!=NULL && *name=='+') {
683 // Converter names that start with '+' are ignored in ICU4J tests.
684 ++name;
685 }
686 if(name!=NULL && *name=='*') {
687 /* loadTestData(): set the data directory */
688 return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
689 } else {
690 return ucnv_open(name, &errorCode);
691 }
692 }
693
694 // output helpers ---------------------------------------------------------- ***
695
696 static inline char
hexDigit(uint8_t digit)697 hexDigit(uint8_t digit) {
698 return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
699 }
700
701 static char *
printBytes(const uint8_t * bytes,int32_t length,char * out)702 printBytes(const uint8_t *bytes, int32_t length, char *out) {
703 uint8_t b;
704
705 if(length>0) {
706 b=*bytes++;
707 --length;
708 *out++=hexDigit((uint8_t)(b>>4));
709 *out++=hexDigit((uint8_t)(b&0xf));
710 }
711
712 while(length>0) {
713 b=*bytes++;
714 --length;
715 *out++=' ';
716 *out++=hexDigit((uint8_t)(b>>4));
717 *out++=hexDigit((uint8_t)(b&0xf));
718 }
719 *out++=0;
720 return out;
721 }
722
723 static char *
printUnicode(const UChar * unicode,int32_t length,char * out)724 printUnicode(const UChar *unicode, int32_t length, char *out) {
725 UChar32 c;
726 int32_t i;
727
728 for(i=0; i<length;) {
729 if(i>0) {
730 *out++=' ';
731 }
732 U16_NEXT(unicode, i, length, c);
733 // write 4..6 digits
734 if(c>=0x100000) {
735 *out++='1';
736 }
737 if(c>=0x10000) {
738 *out++=hexDigit((uint8_t)((c>>16)&0xf));
739 }
740 *out++=hexDigit((uint8_t)((c>>12)&0xf));
741 *out++=hexDigit((uint8_t)((c>>8)&0xf));
742 *out++=hexDigit((uint8_t)((c>>4)&0xf));
743 *out++=hexDigit((uint8_t)(c&0xf));
744 }
745 *out++=0;
746 return out;
747 }
748
749 static char *
printOffsets(const int32_t * offsets,int32_t length,char * out)750 printOffsets(const int32_t *offsets, int32_t length, char *out) {
751 int32_t i, o, d;
752
753 if(offsets==NULL) {
754 length=0;
755 }
756
757 for(i=0; i<length; ++i) {
758 if(i>0) {
759 *out++=' ';
760 }
761 o=offsets[i];
762
763 // print all offsets with 2 characters each (-x, -9..99, xx)
764 if(o<-9) {
765 *out++='-';
766 *out++='x';
767 } else if(o<0) {
768 *out++='-';
769 *out++=(char)('0'-o);
770 } else if(o<=99) {
771 *out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
772 *out++=(char)('0'+o%10);
773 } else /* o>99 */ {
774 *out++='x';
775 *out++='x';
776 }
777 }
778 *out++=0;
779 return out;
780 }
781
782 // toUnicode test worker functions ----------------------------------------- ***
783
784 static int32_t
stepToUnicode(ConversionCase & cc,UConverter * cnv,UChar * result,int32_t resultCapacity,int32_t * resultOffsets,int32_t step,UErrorCode * pErrorCode)785 stepToUnicode(ConversionCase &cc, UConverter *cnv,
786 UChar *result, int32_t resultCapacity,
787 int32_t *resultOffsets, /* also resultCapacity */
788 int32_t step,
789 UErrorCode *pErrorCode) {
790 const char *source, *sourceLimit, *bytesLimit;
791 UChar *target, *targetLimit, *resultLimit;
792 UBool flush;
793
794 source=(const char *)cc.bytes;
795 target=result;
796 bytesLimit=source+cc.bytesLength;
797 resultLimit=result+resultCapacity;
798
799 if(step>=0) {
800 // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
801 // move only one buffer (in vs. out) at a time to be extra mean
802 // step==0 performs bulk conversion and generates offsets
803
804 // initialize the partial limits for the loop
805 if(step==0) {
806 // use the entire buffers
807 sourceLimit=bytesLimit;
808 targetLimit=resultLimit;
809 flush=cc.finalFlush;
810 } else {
811 // start with empty partial buffers
812 sourceLimit=source;
813 targetLimit=target;
814 flush=FALSE;
815
816 // output offsets only for bulk conversion
817 resultOffsets=NULL;
818 }
819
820 for(;;) {
821 // resetting the opposite conversion direction must not affect this one
822 ucnv_resetFromUnicode(cnv);
823
824 // convert
825 ucnv_toUnicode(cnv,
826 &target, targetLimit,
827 &source, sourceLimit,
828 resultOffsets,
829 flush, pErrorCode);
830
831 // check pointers and errors
832 if(source>sourceLimit || target>targetLimit) {
833 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
834 break;
835 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
836 if(target!=targetLimit) {
837 // buffer overflow must only be set when the target is filled
838 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
839 break;
840 } else if(targetLimit==resultLimit) {
841 // not just a partial overflow
842 break;
843 }
844
845 // the partial target is filled, set a new limit, reset the error and continue
846 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
847 *pErrorCode=U_ZERO_ERROR;
848 } else if(U_FAILURE(*pErrorCode)) {
849 // some other error occurred, done
850 break;
851 } else {
852 if(source!=sourceLimit) {
853 // when no error occurs, then the input must be consumed
854 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
855 break;
856 }
857
858 if(sourceLimit==bytesLimit) {
859 // we are done
860 break;
861 }
862
863 // the partial conversion succeeded, set a new limit and continue
864 sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
865 flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
866 }
867 }
868 } else /* step<0 */ {
869 /*
870 * step==-1: call only ucnv_getNextUChar()
871 * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
872 * if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
873 * else give it at most (-step-2)/2 bytes
874 */
875 UChar32 c;
876
877 // end the loop by getting an index out of bounds error
878 for(;;) {
879 // resetting the opposite conversion direction must not affect this one
880 ucnv_resetFromUnicode(cnv);
881
882 // convert
883 if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
884 sourceLimit=source; // use sourceLimit not as a real limit
885 // but to remember the pre-getNextUChar source pointer
886 c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
887
888 // check pointers and errors
889 if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
890 if(source!=bytesLimit) {
891 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
892 } else {
893 *pErrorCode=U_ZERO_ERROR;
894 }
895 break;
896 } else if(U_FAILURE(*pErrorCode)) {
897 break;
898 }
899 // source may not move if c is from previous overflow
900
901 if(target==resultLimit) {
902 *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
903 break;
904 }
905 if(c<=0xffff) {
906 *target++=(UChar)c;
907 } else {
908 *target++=U16_LEAD(c);
909 if(target==resultLimit) {
910 *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
911 break;
912 }
913 *target++=U16_TRAIL(c);
914 }
915
916 // alternate between -n-1 and -n but leave -1 alone
917 if(step<-1) {
918 ++step;
919 }
920 } else /* step is even */ {
921 // allow only one UChar output
922 targetLimit=target<resultLimit ? target+1 : resultLimit;
923
924 // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
925 // and never output offsets
926 if(step==-2) {
927 sourceLimit=bytesLimit;
928 } else {
929 sourceLimit=source+(-step-2)/2;
930 if(sourceLimit>bytesLimit) {
931 sourceLimit=bytesLimit;
932 }
933 }
934
935 ucnv_toUnicode(cnv,
936 &target, targetLimit,
937 &source, sourceLimit,
938 NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
939
940 // check pointers and errors
941 if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
942 if(target!=targetLimit) {
943 // buffer overflow must only be set when the target is filled
944 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
945 break;
946 } else if(targetLimit==resultLimit) {
947 // not just a partial overflow
948 break;
949 }
950
951 // the partial target is filled, set a new limit and continue
952 *pErrorCode=U_ZERO_ERROR;
953 } else if(U_FAILURE(*pErrorCode)) {
954 // some other error occurred, done
955 break;
956 } else {
957 if(source!=sourceLimit) {
958 // when no error occurs, then the input must be consumed
959 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
960 break;
961 }
962
963 // we are done (flush==TRUE) but we continue, to get the index out of bounds error above
964 }
965
966 --step;
967 }
968 }
969 }
970
971 return (int32_t)(target-result);
972 }
973
974 UBool
ToUnicodeCase(ConversionCase & cc,UConverterToUCallback callback,const char * option)975 ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
976 // open the converter
977 IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
978 LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
979 if(errorCode.isFailure()) {
980 errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
981 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
982 errorCode.reset();
983 return FALSE;
984 }
985
986 // set the callback
987 if(callback!=NULL) {
988 ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
989 if(U_FAILURE(errorCode)) {
990 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
991 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
992 return FALSE;
993 }
994 }
995
996 int32_t resultOffsets[256];
997 UChar result[256];
998 int32_t resultLength;
999 UBool ok;
1000
1001 static const struct {
1002 int32_t step;
1003 const char *name;
1004 } steps[]={
1005 { 0, "bulk" }, // must be first for offsets to be checked
1006 { 1, "step=1" },
1007 { 3, "step=3" },
1008 { 7, "step=7" },
1009 { -1, "getNext" },
1010 { -2, "toU(bulk)+getNext" },
1011 { -3, "getNext+toU(bulk)" },
1012 { -4, "toU(1)+getNext" },
1013 { -5, "getNext+toU(1)" },
1014 { -12, "toU(5)+getNext" },
1015 { -13, "getNext+toU(5)" },
1016 };
1017 int32_t i, step;
1018
1019 ok=TRUE;
1020 for(i=0; i<LENGTHOF(steps) && ok; ++i) {
1021 step=steps[i].step;
1022 if(step<0 && !cc.finalFlush) {
1023 // skip ucnv_getNextUChar() if !finalFlush because
1024 // ucnv_getNextUChar() always implies flush
1025 continue;
1026 }
1027 if(step!=0) {
1028 // bulk test is first, then offsets are not checked any more
1029 cc.offsets=NULL;
1030 }
1031 else {
1032 memset(resultOffsets, -1, LENGTHOF(resultOffsets));
1033 }
1034 memset(result, -1, LENGTHOF(result));
1035 errorCode.reset();
1036 resultLength=stepToUnicode(cc, cnv.getAlias(),
1037 result, LENGTHOF(result),
1038 step==0 ? resultOffsets : NULL,
1039 step, errorCode);
1040 ok=checkToUnicode(
1041 cc, cnv.getAlias(), steps[i].name,
1042 result, resultLength,
1043 cc.offsets!=NULL ? resultOffsets : NULL,
1044 errorCode);
1045 if(errorCode.isFailure() || !cc.finalFlush) {
1046 // reset if an error occurred or we did not flush
1047 // otherwise do nothing to make sure that flushing resets
1048 ucnv_resetToUnicode(cnv.getAlias());
1049 }
1050 if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
1051 errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1052 cc.caseNr, cc.charset, resultLength);
1053 }
1054 if (result[resultLength] != (UChar)-1) {
1055 errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
1056 cc.caseNr, cc.charset, resultLength);
1057 }
1058 }
1059
1060 // not a real loop, just a convenience for breaking out of the block
1061 while(ok && cc.finalFlush) {
1062 // test ucnv_toUChars()
1063 memset(result, 0, sizeof(result));
1064
1065 errorCode.reset();
1066 resultLength=ucnv_toUChars(cnv.getAlias(),
1067 result, LENGTHOF(result),
1068 (const char *)cc.bytes, cc.bytesLength,
1069 errorCode);
1070 ok=checkToUnicode(
1071 cc, cnv.getAlias(), "toUChars",
1072 result, resultLength,
1073 NULL,
1074 errorCode);
1075 if(!ok) {
1076 break;
1077 }
1078
1079 // test preflighting
1080 // keep the correct result for simple checking
1081 errorCode.reset();
1082 resultLength=ucnv_toUChars(cnv.getAlias(),
1083 NULL, 0,
1084 (const char *)cc.bytes, cc.bytesLength,
1085 errorCode);
1086 if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
1087 errorCode.reset();
1088 }
1089 ok=checkToUnicode(
1090 cc, cnv.getAlias(), "preflight toUChars",
1091 result, resultLength,
1092 NULL,
1093 errorCode);
1094 break;
1095 }
1096
1097 errorCode.reset(); // all errors have already been reported
1098 return ok;
1099 }
1100
1101 UBool
checkToUnicode(ConversionCase & cc,UConverter * cnv,const char * name,const UChar * result,int32_t resultLength,const int32_t * resultOffsets,UErrorCode resultErrorCode)1102 ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1103 const UChar *result, int32_t resultLength,
1104 const int32_t *resultOffsets,
1105 UErrorCode resultErrorCode) {
1106 char resultInvalidChars[8];
1107 int8_t resultInvalidLength;
1108 UErrorCode errorCode;
1109
1110 const char *msg;
1111
1112 // reset the message; NULL will mean "ok"
1113 msg=NULL;
1114
1115 errorCode=U_ZERO_ERROR;
1116 resultInvalidLength=sizeof(resultInvalidChars);
1117 ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
1118 if(U_FAILURE(errorCode)) {
1119 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
1120 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1121 return FALSE;
1122 }
1123
1124 // check everything that might have gone wrong
1125 if(cc.unicodeLength!=resultLength) {
1126 msg="wrong result length";
1127 } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
1128 msg="wrong result string";
1129 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
1130 msg="wrong offsets";
1131 } else if(cc.outErrorCode!=resultErrorCode) {
1132 msg="wrong error code";
1133 } else if(cc.invalidLength!=resultInvalidLength) {
1134 msg="wrong length of last invalid input";
1135 } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
1136 msg="wrong last invalid input";
1137 }
1138
1139 if(msg==NULL) {
1140 return TRUE;
1141 } else {
1142 char buffer[2000]; // one buffer for all strings
1143 char *s, *bytesString, *unicodeString, *resultString,
1144 *offsetsString, *resultOffsetsString,
1145 *invalidCharsString, *resultInvalidCharsString;
1146
1147 bytesString=s=buffer;
1148 s=printBytes(cc.bytes, cc.bytesLength, bytesString);
1149 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
1150 s=printUnicode(result, resultLength, resultString=s);
1151 s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
1152 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1153 s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
1154 s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
1155
1156 if((s-buffer)>(int32_t)sizeof(buffer)) {
1157 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
1158 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1159 exit(1);
1160 }
1161
1162 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1163 " bytes <%s>[%d]\n"
1164 " expected <%s>[%d]\n"
1165 " result <%s>[%d]\n"
1166 " offsets <%s>\n"
1167 " result offsets <%s>\n"
1168 " error code expected %s got %s\n"
1169 " invalidChars expected <%s> got <%s>\n",
1170 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1171 bytesString, cc.bytesLength,
1172 unicodeString, cc.unicodeLength,
1173 resultString, resultLength,
1174 offsetsString,
1175 resultOffsetsString,
1176 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1177 invalidCharsString, resultInvalidCharsString);
1178
1179 return FALSE;
1180 }
1181 }
1182
1183 // fromUnicode test worker functions --------------------------------------- ***
1184
1185 static int32_t
stepFromUTF8(ConversionCase & cc,UConverter * utf8Cnv,UConverter * cnv,char * result,int32_t resultCapacity,int32_t step,UErrorCode * pErrorCode)1186 stepFromUTF8(ConversionCase &cc,
1187 UConverter *utf8Cnv, UConverter *cnv,
1188 char *result, int32_t resultCapacity,
1189 int32_t step,
1190 UErrorCode *pErrorCode) {
1191 const char *source, *sourceLimit, *utf8Limit;
1192 UChar pivotBuffer[32];
1193 UChar *pivotSource, *pivotTarget, *pivotLimit;
1194 char *target, *targetLimit, *resultLimit;
1195 UBool flush;
1196
1197 source=cc.utf8;
1198 pivotSource=pivotTarget=pivotBuffer;
1199 target=result;
1200 utf8Limit=source+cc.utf8Length;
1201 resultLimit=result+resultCapacity;
1202
1203 // call ucnv_convertEx() with in/out buffers no larger than (step) at a time
1204 // move only one buffer (in vs. out) at a time to be extra mean
1205 // step==0 performs bulk conversion
1206
1207 // initialize the partial limits for the loop
1208 if(step==0) {
1209 // use the entire buffers
1210 sourceLimit=utf8Limit;
1211 targetLimit=resultLimit;
1212 flush=cc.finalFlush;
1213
1214 pivotLimit=pivotBuffer+LENGTHOF(pivotBuffer);
1215 } else {
1216 // start with empty partial buffers
1217 sourceLimit=source;
1218 targetLimit=target;
1219 flush=FALSE;
1220
1221 // empty pivot is not allowed, make it of length step
1222 pivotLimit=pivotBuffer+step;
1223 }
1224
1225 for(;;) {
1226 // resetting the opposite conversion direction must not affect this one
1227 ucnv_resetFromUnicode(utf8Cnv);
1228 ucnv_resetToUnicode(cnv);
1229
1230 // convert
1231 ucnv_convertEx(cnv, utf8Cnv,
1232 &target, targetLimit,
1233 &source, sourceLimit,
1234 pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
1235 FALSE, flush, pErrorCode);
1236
1237 // check pointers and errors
1238 if(source>sourceLimit || target>targetLimit) {
1239 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1240 break;
1241 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1242 if(target!=targetLimit) {
1243 // buffer overflow must only be set when the target is filled
1244 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1245 break;
1246 } else if(targetLimit==resultLimit) {
1247 // not just a partial overflow
1248 break;
1249 }
1250
1251 // the partial target is filled, set a new limit, reset the error and continue
1252 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1253 *pErrorCode=U_ZERO_ERROR;
1254 } else if(U_FAILURE(*pErrorCode)) {
1255 if(pivotSource==pivotBuffer) {
1256 // toUnicode error, should not occur
1257 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1258 break;
1259 } else {
1260 // fromUnicode error
1261 // some other error occurred, done
1262 break;
1263 }
1264 } else {
1265 if(source!=sourceLimit) {
1266 // when no error occurs, then the input must be consumed
1267 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1268 break;
1269 }
1270
1271 if(sourceLimit==utf8Limit) {
1272 // we are done
1273 if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
1274 // ucnv_convertEx() warns about not terminating the output
1275 // but ucnv_fromUnicode() does not and so
1276 // checkFromUnicode() does not expect it
1277 *pErrorCode=U_ZERO_ERROR;
1278 }
1279 break;
1280 }
1281
1282 // the partial conversion succeeded, set a new limit and continue
1283 sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
1284 flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
1285 }
1286 }
1287
1288 return (int32_t)(target-result);
1289 }
1290
1291 static int32_t
stepFromUnicode(ConversionCase & cc,UConverter * cnv,char * result,int32_t resultCapacity,int32_t * resultOffsets,int32_t step,UErrorCode * pErrorCode)1292 stepFromUnicode(ConversionCase &cc, UConverter *cnv,
1293 char *result, int32_t resultCapacity,
1294 int32_t *resultOffsets, /* also resultCapacity */
1295 int32_t step,
1296 UErrorCode *pErrorCode) {
1297 const UChar *source, *sourceLimit, *unicodeLimit;
1298 char *target, *targetLimit, *resultLimit;
1299 UBool flush;
1300
1301 source=cc.unicode;
1302 target=result;
1303 unicodeLimit=source+cc.unicodeLength;
1304 resultLimit=result+resultCapacity;
1305
1306 // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
1307 // move only one buffer (in vs. out) at a time to be extra mean
1308 // step==0 performs bulk conversion and generates offsets
1309
1310 // initialize the partial limits for the loop
1311 if(step==0) {
1312 // use the entire buffers
1313 sourceLimit=unicodeLimit;
1314 targetLimit=resultLimit;
1315 flush=cc.finalFlush;
1316 } else {
1317 // start with empty partial buffers
1318 sourceLimit=source;
1319 targetLimit=target;
1320 flush=FALSE;
1321
1322 // output offsets only for bulk conversion
1323 resultOffsets=NULL;
1324 }
1325
1326 for(;;) {
1327 // resetting the opposite conversion direction must not affect this one
1328 ucnv_resetToUnicode(cnv);
1329
1330 // convert
1331 ucnv_fromUnicode(cnv,
1332 &target, targetLimit,
1333 &source, sourceLimit,
1334 resultOffsets,
1335 flush, pErrorCode);
1336
1337 // check pointers and errors
1338 if(source>sourceLimit || target>targetLimit) {
1339 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1340 break;
1341 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1342 if(target!=targetLimit) {
1343 // buffer overflow must only be set when the target is filled
1344 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1345 break;
1346 } else if(targetLimit==resultLimit) {
1347 // not just a partial overflow
1348 break;
1349 }
1350
1351 // the partial target is filled, set a new limit, reset the error and continue
1352 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1353 *pErrorCode=U_ZERO_ERROR;
1354 } else if(U_FAILURE(*pErrorCode)) {
1355 // some other error occurred, done
1356 break;
1357 } else {
1358 if(source!=sourceLimit) {
1359 // when no error occurs, then the input must be consumed
1360 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1361 break;
1362 }
1363
1364 if(sourceLimit==unicodeLimit) {
1365 // we are done
1366 break;
1367 }
1368
1369 // the partial conversion succeeded, set a new limit and continue
1370 sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
1371 flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
1372 }
1373 }
1374
1375 return (int32_t)(target-result);
1376 }
1377
1378 UBool
FromUnicodeCase(ConversionCase & cc,UConverterFromUCallback callback,const char * option)1379 ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
1380 UConverter *cnv;
1381 UErrorCode errorCode;
1382
1383 // open the converter
1384 errorCode=U_ZERO_ERROR;
1385 cnv=cnv_open(cc.charset, errorCode);
1386 if(U_FAILURE(errorCode)) {
1387 errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1388 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1389 return FALSE;
1390 }
1391 ucnv_resetToUnicode(utf8Cnv);
1392
1393 // set the callback
1394 if(callback!=NULL) {
1395 ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
1396 if(U_FAILURE(errorCode)) {
1397 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
1398 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1399 ucnv_close(cnv);
1400 return FALSE;
1401 }
1402 }
1403
1404 // set the fallbacks flag
1405 // TODO change with Jitterbug 2401, then add a similar call for toUnicode too
1406 ucnv_setFallback(cnv, cc.fallbacks);
1407
1408 // set the subchar
1409 int32_t length;
1410
1411 if(cc.setSub>0) {
1412 length=(int32_t)strlen(cc.subchar);
1413 ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
1414 if(U_FAILURE(errorCode)) {
1415 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
1416 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1417 ucnv_close(cnv);
1418 return FALSE;
1419 }
1420 } else if(cc.setSub<0) {
1421 ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
1422 if(U_FAILURE(errorCode)) {
1423 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
1424 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1425 ucnv_close(cnv);
1426 return FALSE;
1427 }
1428 }
1429
1430 // convert unicode to utf8
1431 char utf8[256];
1432 cc.utf8=utf8;
1433 u_strToUTF8(utf8, LENGTHOF(utf8), &cc.utf8Length,
1434 cc.unicode, cc.unicodeLength,
1435 &errorCode);
1436 if(U_FAILURE(errorCode)) {
1437 // skip UTF-8 testing of a string with an unpaired surrogate,
1438 // or of one that's too long
1439 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1440 cc.utf8Length=-1;
1441 }
1442
1443 int32_t resultOffsets[256];
1444 char result[256];
1445 int32_t resultLength;
1446 UBool ok;
1447
1448 static const struct {
1449 int32_t step;
1450 const char *name, *utf8Name;
1451 } steps[]={
1452 { 0, "bulk", "utf8" }, // must be first for offsets to be checked
1453 { 1, "step=1", "utf8 step=1" },
1454 { 3, "step=3", "utf8 step=3" },
1455 { 7, "step=7", "utf8 step=7" }
1456 };
1457 int32_t i, step;
1458
1459 ok=TRUE;
1460 for(i=0; i<LENGTHOF(steps) && ok; ++i) {
1461 step=steps[i].step;
1462 memset(resultOffsets, -1, LENGTHOF(resultOffsets));
1463 memset(result, -1, LENGTHOF(result));
1464 errorCode=U_ZERO_ERROR;
1465 resultLength=stepFromUnicode(cc, cnv,
1466 result, LENGTHOF(result),
1467 step==0 ? resultOffsets : NULL,
1468 step, &errorCode);
1469 ok=checkFromUnicode(
1470 cc, cnv, steps[i].name,
1471 (uint8_t *)result, resultLength,
1472 cc.offsets!=NULL ? resultOffsets : NULL,
1473 errorCode);
1474 if(U_FAILURE(errorCode) || !cc.finalFlush) {
1475 // reset if an error occurred or we did not flush
1476 // otherwise do nothing to make sure that flushing resets
1477 ucnv_resetFromUnicode(cnv);
1478 }
1479 if (resultOffsets[resultLength] != -1) {
1480 errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1481 cc.caseNr, cc.charset, resultLength);
1482 }
1483 if (result[resultLength] != (char)-1) {
1484 errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
1485 cc.caseNr, cc.charset, resultLength);
1486 }
1487
1488 // bulk test is first, then offsets are not checked any more
1489 cc.offsets=NULL;
1490
1491 // test direct conversion from UTF-8
1492 if(cc.utf8Length>=0) {
1493 errorCode=U_ZERO_ERROR;
1494 resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
1495 result, LENGTHOF(result),
1496 step, &errorCode);
1497 ok=checkFromUnicode(
1498 cc, cnv, steps[i].utf8Name,
1499 (uint8_t *)result, resultLength,
1500 NULL,
1501 errorCode);
1502 if(U_FAILURE(errorCode) || !cc.finalFlush) {
1503 // reset if an error occurred or we did not flush
1504 // otherwise do nothing to make sure that flushing resets
1505 ucnv_resetToUnicode(utf8Cnv);
1506 ucnv_resetFromUnicode(cnv);
1507 }
1508 }
1509 }
1510
1511 // not a real loop, just a convenience for breaking out of the block
1512 while(ok && cc.finalFlush) {
1513 // test ucnv_fromUChars()
1514 memset(result, 0, sizeof(result));
1515
1516 errorCode=U_ZERO_ERROR;
1517 resultLength=ucnv_fromUChars(cnv,
1518 result, LENGTHOF(result),
1519 cc.unicode, cc.unicodeLength,
1520 &errorCode);
1521 ok=checkFromUnicode(
1522 cc, cnv, "fromUChars",
1523 (uint8_t *)result, resultLength,
1524 NULL,
1525 errorCode);
1526 if(!ok) {
1527 break;
1528 }
1529
1530 // test preflighting
1531 // keep the correct result for simple checking
1532 errorCode=U_ZERO_ERROR;
1533 resultLength=ucnv_fromUChars(cnv,
1534 NULL, 0,
1535 cc.unicode, cc.unicodeLength,
1536 &errorCode);
1537 if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
1538 errorCode=U_ZERO_ERROR;
1539 }
1540 ok=checkFromUnicode(
1541 cc, cnv, "preflight fromUChars",
1542 (uint8_t *)result, resultLength,
1543 NULL,
1544 errorCode);
1545 break;
1546 }
1547
1548 ucnv_close(cnv);
1549 return ok;
1550 }
1551
1552 UBool
checkFromUnicode(ConversionCase & cc,UConverter * cnv,const char * name,const uint8_t * result,int32_t resultLength,const int32_t * resultOffsets,UErrorCode resultErrorCode)1553 ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1554 const uint8_t *result, int32_t resultLength,
1555 const int32_t *resultOffsets,
1556 UErrorCode resultErrorCode) {
1557 UChar resultInvalidUChars[8];
1558 int8_t resultInvalidLength;
1559 UErrorCode errorCode;
1560
1561 const char *msg;
1562
1563 // reset the message; NULL will mean "ok"
1564 msg=NULL;
1565
1566 errorCode=U_ZERO_ERROR;
1567 resultInvalidLength=LENGTHOF(resultInvalidUChars);
1568 ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
1569 if(U_FAILURE(errorCode)) {
1570 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
1571 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1572 return FALSE;
1573 }
1574
1575 // check everything that might have gone wrong
1576 if(cc.bytesLength!=resultLength) {
1577 msg="wrong result length";
1578 } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
1579 msg="wrong result string";
1580 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
1581 msg="wrong offsets";
1582 } else if(cc.outErrorCode!=resultErrorCode) {
1583 msg="wrong error code";
1584 } else if(cc.invalidLength!=resultInvalidLength) {
1585 msg="wrong length of last invalid input";
1586 } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
1587 msg="wrong last invalid input";
1588 }
1589
1590 if(msg==NULL) {
1591 return TRUE;
1592 } else {
1593 char buffer[2000]; // one buffer for all strings
1594 char *s, *unicodeString, *bytesString, *resultString,
1595 *offsetsString, *resultOffsetsString,
1596 *invalidCharsString, *resultInvalidUCharsString;
1597
1598 unicodeString=s=buffer;
1599 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
1600 s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
1601 s=printBytes(result, resultLength, resultString=s);
1602 s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
1603 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1604 s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
1605 s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
1606
1607 if((s-buffer)>(int32_t)sizeof(buffer)) {
1608 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
1609 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1610 exit(1);
1611 }
1612
1613 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1614 " unicode <%s>[%d]\n"
1615 " expected <%s>[%d]\n"
1616 " result <%s>[%d]\n"
1617 " offsets <%s>\n"
1618 " result offsets <%s>\n"
1619 " error code expected %s got %s\n"
1620 " invalidChars expected <%s> got <%s>\n",
1621 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1622 unicodeString, cc.unicodeLength,
1623 bytesString, cc.bytesLength,
1624 resultString, resultLength,
1625 offsetsString,
1626 resultOffsetsString,
1627 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1628 invalidCharsString, resultInvalidUCharsString);
1629
1630 return FALSE;
1631 }
1632 }
1633
1634 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
1635