1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 *******************************************************************************
5 *
6 * Copyright (C) 2003-2014, International Business Machines
7 * Corporation and others. All Rights Reserved.
8 *
9 *******************************************************************************
10 * file name: convtest.cpp
11 * encoding: UTF-8
12 * tab size: 8 (not used)
13 * indentation:4
14 *
15 * created on: 2003jul15
16 * created by: Markus W. Scherer
17 *
18 * Test file for data-driven conversion tests.
19 */
20
21 #include "unicode/utypes.h"
22
23 #if !UCONFIG_NO_LEGACY_CONVERSION
24 /*
25 * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
26 * is slightly unnecessary - it removes tests for Unicode charsets
27 * like UTF-8 that should work.
28 * However, there is no easy way for the test to detect whether a test case
29 * is for a Unicode charset, so it would be difficult to only exclude those.
30 * Also, regular testing of ICU is done with all modules on, therefore
31 * not testing conversion for a custom configuration like this should be ok.
32 */
33
34 #include "unicode/ucnv.h"
35 #include "unicode/unistr.h"
36 #include "unicode/parsepos.h"
37 #include "unicode/uniset.h"
38 #include "unicode/usetiter.h"
39 #include "unicode/ustring.h"
40 #include "unicode/ures.h"
41 #include "unicode/utf16.h"
42 #include "convtest.h"
43 #include "cmemory.h"
44 #include "unicode/tstdtmod.h"
45 #include <string.h>
46 #include <stdlib.h>
47
48 enum {
49 // characters used in test data for callbacks
50 SUB_CB='?',
51 SKIP_CB='0',
52 STOP_CB='.',
53 ESC_CB='&'
54 };
55
ConversionTest()56 ConversionTest::ConversionTest() {
57 UErrorCode errorCode=U_ZERO_ERROR;
58 utf8Cnv=ucnv_open("UTF-8", &errorCode);
59 ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
60 if(U_FAILURE(errorCode)) {
61 errln("unable to open UTF-8 converter");
62 }
63 }
64
~ConversionTest()65 ConversionTest::~ConversionTest() {
66 ucnv_close(utf8Cnv);
67 }
68
69 void
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)70 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
71 if (exec) logln("TestSuite ConversionTest: ");
72 TESTCASE_AUTO_BEGIN;
73 #if !UCONFIG_NO_FILE_IO
74 TESTCASE_AUTO(TestToUnicode);
75 TESTCASE_AUTO(TestFromUnicode);
76 TESTCASE_AUTO(TestGetUnicodeSet);
77 #endif
78 TESTCASE_AUTO(TestGetUnicodeSet2);
79 TESTCASE_AUTO(TestDefaultIgnorableCallback);
80 TESTCASE_AUTO(TestUTF8ToUTF8Overflow);
81 TESTCASE_AUTO(TestUTF8ToUTF8Streaming);
82 TESTCASE_AUTO_END;
83 }
84
85 // test data interface ----------------------------------------------------- ***
86
87 void
TestToUnicode()88 ConversionTest::TestToUnicode() {
89 ConversionCase cc;
90 char charset[100], cbopt[4];
91 const char *option;
92 UnicodeString s, unicode;
93 int32_t offsetsLength;
94 UConverterToUCallback callback;
95
96 TestDataModule *dataModule;
97 TestData *testData;
98 const DataMap *testCase;
99 UErrorCode errorCode;
100 int32_t i;
101
102 errorCode=U_ZERO_ERROR;
103 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
104 if(U_SUCCESS(errorCode)) {
105 testData=dataModule->createTestData("toUnicode", errorCode);
106 if(U_SUCCESS(errorCode)) {
107 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
108 if(U_FAILURE(errorCode)) {
109 errln("error retrieving conversion/toUnicode test case %d - %s",
110 i, u_errorName(errorCode));
111 errorCode=U_ZERO_ERROR;
112 continue;
113 }
114
115 cc.caseNr=i;
116
117 s=testCase->getString("charset", errorCode);
118 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
119 cc.charset=charset;
120
121 // BEGIN android-added
122 // To save space, Android does not build full ISO-2022-CN tables.
123 // We skip the TestGetKeywordValuesForLocale for counting available collations.
124 if (strlen(charset) >= 8 &&
125 strncmp(charset+4, "2022-CN", 4) == 0) {
126 continue;
127 }
128 // END android-added
129
130 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
131 unicode=testCase->getString("unicode", errorCode);
132 cc.unicode=unicode.getBuffer();
133 cc.unicodeLength=unicode.length();
134
135 offsetsLength=0;
136 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
137 if(offsetsLength==0) {
138 cc.offsets=NULL;
139 } else if(offsetsLength!=unicode.length()) {
140 errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
141 i, unicode.length(), offsetsLength);
142 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
143 }
144
145 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
146 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
147
148 s=testCase->getString("errorCode", errorCode);
149 if(s==UNICODE_STRING("invalid", 7)) {
150 cc.outErrorCode=U_INVALID_CHAR_FOUND;
151 } else if(s==UNICODE_STRING("illegal", 7)) {
152 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
153 } else if(s==UNICODE_STRING("truncated", 9)) {
154 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
155 } else if(s==UNICODE_STRING("illesc", 6)) {
156 cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
157 } else if(s==UNICODE_STRING("unsuppesc", 9)) {
158 cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
159 } else {
160 cc.outErrorCode=U_ZERO_ERROR;
161 }
162
163 s=testCase->getString("callback", errorCode);
164 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
165 cc.cbopt=cbopt;
166 switch(cbopt[0]) {
167 case SUB_CB:
168 callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
169 break;
170 case SKIP_CB:
171 callback=UCNV_TO_U_CALLBACK_SKIP;
172 break;
173 case STOP_CB:
174 callback=UCNV_TO_U_CALLBACK_STOP;
175 break;
176 case ESC_CB:
177 callback=UCNV_TO_U_CALLBACK_ESCAPE;
178 break;
179 default:
180 callback=NULL;
181 break;
182 }
183 option=callback==NULL ? cbopt : cbopt+1;
184 if(*option==0) {
185 option=NULL;
186 }
187
188 cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
189
190 if(U_FAILURE(errorCode)) {
191 errln("error parsing conversion/toUnicode test case %d - %s",
192 i, u_errorName(errorCode));
193 errorCode=U_ZERO_ERROR;
194 } else {
195 logln("TestToUnicode[%d] %s", i, charset);
196 ToUnicodeCase(cc, callback, option);
197 }
198 }
199 delete testData;
200 }
201 delete dataModule;
202 }
203 else {
204 dataerrln("Could not load test conversion data");
205 }
206 }
207
208 void
TestFromUnicode()209 ConversionTest::TestFromUnicode() {
210 ConversionCase cc;
211 char charset[100], cbopt[4];
212 const char *option;
213 UnicodeString s, unicode, invalidUChars;
214 int32_t offsetsLength, index;
215 UConverterFromUCallback callback;
216
217 TestDataModule *dataModule;
218 TestData *testData;
219 const DataMap *testCase;
220 const UChar *p;
221 UErrorCode errorCode;
222 int32_t i, length;
223
224 errorCode=U_ZERO_ERROR;
225 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
226 if(U_SUCCESS(errorCode)) {
227 testData=dataModule->createTestData("fromUnicode", errorCode);
228 if(U_SUCCESS(errorCode)) {
229 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
230 if(U_FAILURE(errorCode)) {
231 errln("error retrieving conversion/fromUnicode test case %d - %s",
232 i, u_errorName(errorCode));
233 errorCode=U_ZERO_ERROR;
234 continue;
235 }
236
237 cc.caseNr=i;
238
239 s=testCase->getString("charset", errorCode);
240 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
241 cc.charset=charset;
242
243 // BEGIN android-added
244 // To save space, Android does not build full ISO-2022-CN tables.
245 // We skip the TestGetKeywordValuesForLocale for counting available collations.
246 if (strlen(charset) >= 8 &&
247 strncmp(charset+4, "2022-CN", 4) == 0) {
248 continue;
249 }
250 // END android-added
251
252 unicode=testCase->getString("unicode", errorCode);
253 cc.unicode=unicode.getBuffer();
254 cc.unicodeLength=unicode.length();
255 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
256
257 offsetsLength=0;
258 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
259 if(offsetsLength==0) {
260 cc.offsets=NULL;
261 } else if(offsetsLength!=cc.bytesLength) {
262 errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
263 i, cc.bytesLength, offsetsLength);
264 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
265 }
266
267 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
268 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
269
270 s=testCase->getString("errorCode", errorCode);
271 if(s==UNICODE_STRING("invalid", 7)) {
272 cc.outErrorCode=U_INVALID_CHAR_FOUND;
273 } else if(s==UNICODE_STRING("illegal", 7)) {
274 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
275 } else if(s==UNICODE_STRING("truncated", 9)) {
276 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
277 } else {
278 cc.outErrorCode=U_ZERO_ERROR;
279 }
280
281 s=testCase->getString("callback", errorCode);
282 cc.setSub=0; // default: no subchar
283
284 if((index=s.indexOf((UChar)0))>0) {
285 // read NUL-separated subchar first, if any
286 // copy the subchar from Latin-1 characters
287 // start after the NUL
288 p=s.getTerminatedBuffer();
289 length=index+1;
290 p+=length;
291 length=s.length()-length;
292 if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
293 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
294 } else {
295 int32_t j;
296
297 for(j=0; j<length; ++j) {
298 cc.subchar[j]=(char)p[j];
299 }
300 // NUL-terminate the subchar
301 cc.subchar[j]=0;
302 cc.setSub=1;
303 }
304
305 // remove the NUL and subchar from s
306 s.truncate(index);
307 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
308 // read a substitution string, separated by an equal sign
309 p=s.getBuffer()+index+1;
310 length=s.length()-(index+1);
311 if(length<0 || length>=UPRV_LENGTHOF(cc.subString)) {
312 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
313 } else {
314 u_memcpy(cc.subString, p, length);
315 // NUL-terminate the subString
316 cc.subString[length]=0;
317 cc.setSub=-1;
318 }
319
320 // remove the equal sign and subString from s
321 s.truncate(index);
322 }
323
324 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
325 cc.cbopt=cbopt;
326 switch(cbopt[0]) {
327 case SUB_CB:
328 callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
329 break;
330 case SKIP_CB:
331 callback=UCNV_FROM_U_CALLBACK_SKIP;
332 break;
333 case STOP_CB:
334 callback=UCNV_FROM_U_CALLBACK_STOP;
335 break;
336 case ESC_CB:
337 callback=UCNV_FROM_U_CALLBACK_ESCAPE;
338 break;
339 default:
340 callback=NULL;
341 break;
342 }
343 option=callback==NULL ? cbopt : cbopt+1;
344 if(*option==0) {
345 option=NULL;
346 }
347
348 invalidUChars=testCase->getString("invalidUChars", errorCode);
349 cc.invalidUChars=invalidUChars.getBuffer();
350 cc.invalidLength=invalidUChars.length();
351
352 if(U_FAILURE(errorCode)) {
353 errln("error parsing conversion/fromUnicode test case %d - %s",
354 i, u_errorName(errorCode));
355 errorCode=U_ZERO_ERROR;
356 } else {
357 logln("TestFromUnicode[%d] %s", i, charset);
358 FromUnicodeCase(cc, callback, option);
359 }
360 }
361 delete testData;
362 }
363 delete dataModule;
364 }
365 else {
366 dataerrln("Could not load test conversion data");
367 }
368 }
369
370 static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
371
372 void
TestGetUnicodeSet()373 ConversionTest::TestGetUnicodeSet() {
374 char charset[100];
375 UnicodeString s, map, mapnot;
376 int32_t which;
377
378 ParsePosition pos;
379 UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
380 UnicodeSet *cnvSetPtr = &cnvSet;
381 LocalUConverterPointer cnv;
382
383 TestDataModule *dataModule;
384 TestData *testData;
385 const DataMap *testCase;
386 UErrorCode errorCode;
387 int32_t i;
388
389 errorCode=U_ZERO_ERROR;
390 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
391 if(U_SUCCESS(errorCode)) {
392 testData=dataModule->createTestData("getUnicodeSet", errorCode);
393 if(U_SUCCESS(errorCode)) {
394 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
395 if(U_FAILURE(errorCode)) {
396 errln("error retrieving conversion/getUnicodeSet test case %d - %s",
397 i, u_errorName(errorCode));
398 errorCode=U_ZERO_ERROR;
399 continue;
400 }
401
402 s=testCase->getString("charset", errorCode);
403 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
404
405 // BEGIN android-added
406 // To save space, Android does not build full ISO-2022-CN tables.
407 // We skip the TestGetKeywordValuesForLocale for counting available collations.
408 if (strlen(charset) >= 8 &&
409 strncmp(charset+4, "2022-CN", 4) == 0) {
410 continue;
411 }
412 // END android-added
413
414 map=testCase->getString("map", errorCode);
415 mapnot=testCase->getString("mapnot", errorCode);
416
417 which=testCase->getInt28("which", errorCode);
418
419 if(U_FAILURE(errorCode)) {
420 errln("error parsing conversion/getUnicodeSet test case %d - %s",
421 i, u_errorName(errorCode));
422 errorCode=U_ZERO_ERROR;
423 continue;
424 }
425
426 // test this test case
427 mapSet.clear();
428 mapnotSet.clear();
429
430 pos.setIndex(0);
431 mapSet.applyPattern(map, pos, 0, NULL, errorCode);
432 if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
433 errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
434 " error index %d index %d U+%04x",
435 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
436 errorCode=U_ZERO_ERROR;
437 continue;
438 }
439
440 pos.setIndex(0);
441 mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
442 if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
443 errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
444 " error index %d index %d U+%04x",
445 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
446 errorCode=U_ZERO_ERROR;
447 continue;
448 }
449
450 logln("TestGetUnicodeSet[%d] %s", i, charset);
451
452 cnv.adoptInstead(cnv_open(charset, errorCode));
453 if(U_FAILURE(errorCode)) {
454 errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
455 charset, i, u_errorName(errorCode));
456 errorCode=U_ZERO_ERROR;
457 continue;
458 }
459
460 ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
461
462 if(U_FAILURE(errorCode)) {
463 errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
464 charset, i, u_errorName(errorCode));
465 errorCode=U_ZERO_ERROR;
466 continue;
467 }
468
469 // are there items that must be in cnvSet but are not?
470 (diffSet=mapSet).removeAll(cnvSet);
471 if(!diffSet.isEmpty()) {
472 diffSet.toPattern(s, TRUE);
473 if(s.length()>100) {
474 s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
475 }
476 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
477 charset, i);
478 errln(s);
479 }
480
481 // are there items that must not be in cnvSet but are?
482 (diffSet=mapnotSet).retainAll(cnvSet);
483 if(!diffSet.isEmpty()) {
484 diffSet.toPattern(s, TRUE);
485 if(s.length()>100) {
486 s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
487 }
488 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
489 charset, i);
490 errln(s);
491 }
492 }
493 delete testData;
494 }
495 delete dataModule;
496 }
497 else {
498 dataerrln("Could not load test conversion data");
499 }
500 }
501
502 U_CDECL_BEGIN
503 static void U_CALLCONV
getUnicodeSetCallback(const void * context,UConverterFromUnicodeArgs *,const UChar *,int32_t,UChar32 codePoint,UConverterCallbackReason reason,UErrorCode * pErrorCode)504 getUnicodeSetCallback(const void *context,
505 UConverterFromUnicodeArgs * /*fromUArgs*/,
506 const UChar* /*codeUnits*/,
507 int32_t /*length*/,
508 UChar32 codePoint,
509 UConverterCallbackReason reason,
510 UErrorCode *pErrorCode) {
511 if(reason<=UCNV_IRREGULAR) {
512 ((UnicodeSet *)context)->remove(codePoint); // the converter cannot convert this code point
513 *pErrorCode=U_ZERO_ERROR; // skip
514 } // else ignore the reset, close and clone calls.
515 }
516 U_CDECL_END
517
518 // Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
519 void
TestGetUnicodeSet2()520 ConversionTest::TestGetUnicodeSet2() {
521 // Build a string with all code points.
522 UChar32 cpLimit;
523 int32_t s0Length;
524 if(quick) {
525 cpLimit=s0Length=0x10000; // BMP only
526 } else {
527 cpLimit=0x110000;
528 s0Length=0x10000+0x200000; // BMP + surrogate pairs
529 }
530 UChar *s0=new UChar[s0Length];
531 if(s0==NULL) {
532 return;
533 }
534 UChar *s=s0;
535 UChar32 c;
536 UChar c2;
537 // low BMP
538 for(c=0; c<=0xd7ff; ++c) {
539 *s++=(UChar)c;
540 }
541 // trail surrogates
542 for(c=0xdc00; c<=0xdfff; ++c) {
543 *s++=(UChar)c;
544 }
545 // lead surrogates
546 // (after trails so that there is not even one surrogate pair in between)
547 for(c=0xd800; c<=0xdbff; ++c) {
548 *s++=(UChar)c;
549 }
550 // high BMP
551 for(c=0xe000; c<=0xffff; ++c) {
552 *s++=(UChar)c;
553 }
554 // supplementary code points = surrogate pairs
555 if(cpLimit==0x110000) {
556 for(c=0xd800; c<=0xdbff; ++c) {
557 for(c2=0xdc00; c2<=0xdfff; ++c2) {
558 *s++=(UChar)c;
559 *s++=c2;
560 }
561 }
562 }
563
564 static const char *const cnvNames[]={
565 "UTF-8",
566 "UTF-7",
567 "UTF-16",
568 "US-ASCII",
569 "ISO-8859-1",
570 "windows-1252",
571 "Shift-JIS",
572 "ibm-1390", // EBCDIC_STATEFUL table
573 "ibm-16684", // DBCS-only extension table based on EBCDIC_STATEFUL table
574 "HZ",
575 "ISO-2022-JP",
576 "JIS7",
577 "ISO-2022-CN",
578 "ISO-2022-CN-EXT",
579 "LMBCS"
580 };
581 LocalUConverterPointer cnv;
582 char buffer[1024];
583 int32_t i;
584 for(i=0; i<UPRV_LENGTHOF(cnvNames); ++i) {
585 UErrorCode errorCode=U_ZERO_ERROR;
586 cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
587 if(U_FAILURE(errorCode)) {
588 errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
589 continue;
590 }
591 UnicodeSet expected;
592 ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
593 if(U_FAILURE(errorCode)) {
594 errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
595 continue;
596 }
597 UConverterUnicodeSet which;
598 for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
599 if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
600 ucnv_setFallback(cnv.getAlias(), TRUE);
601 }
602 expected.add(0, cpLimit-1);
603 s=s0;
604 UBool flush;
605 do {
606 char *t=buffer;
607 flush=(UBool)(s==s0+s0Length);
608 ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
609 if(U_FAILURE(errorCode)) {
610 if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
611 errorCode=U_ZERO_ERROR;
612 continue;
613 } else {
614 break; // unexpected error, should not occur
615 }
616 }
617 } while(!flush);
618 UnicodeSet set;
619 ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
620 if(cpLimit<0x110000) {
621 set.remove(cpLimit, 0x10ffff);
622 }
623 if(which==UCNV_ROUNDTRIP_SET) {
624 // ignore PUA code points because they will be converted even if they
625 // are fallbacks and when other fallbacks are turned off,
626 // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
627 expected.remove(0xe000, 0xf8ff);
628 expected.remove(0xf0000, 0xffffd);
629 expected.remove(0x100000, 0x10fffd);
630 set.remove(0xe000, 0xf8ff);
631 set.remove(0xf0000, 0xffffd);
632 set.remove(0x100000, 0x10fffd);
633 }
634 if(set!=expected) {
635 // First try to see if we have different sets because ucnv_getUnicodeSet()
636 // added strings: The above conversion method does not tell us what strings might be convertible.
637 // Remove strings from the set and compare again.
638 set.removeAllStrings();
639 }
640 if(set!=expected) {
641 UnicodeSet diffSet;
642 UnicodeString out;
643
644 // are there items that must be in the set but are not?
645 (diffSet=expected).removeAll(set);
646 if(!diffSet.isEmpty()) {
647 diffSet.toPattern(out, TRUE);
648 if(out.length()>100) {
649 out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
650 }
651 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
652 cnvNames[i], which);
653 errln(out);
654 }
655
656 // are there items that must not be in the set but are?
657 (diffSet=set).removeAll(expected);
658 if(!diffSet.isEmpty()) {
659 diffSet.toPattern(out, TRUE);
660 if(out.length()>100) {
661 out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
662 }
663 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
664 cnvNames[i], which);
665 errln(out);
666 }
667 }
668 }
669 }
670
671 delete [] s0;
672 }
673
674 // Test that all code points which have the default ignorable Unicode property
675 // are ignored if they have no mapping.
676 // If there are any failures, the hard coded list (IS_DEFAULT_IGNORABLE_CODE_POINT)
677 // in ucnv_err.cpp should be updated.
678 void
TestDefaultIgnorableCallback()679 ConversionTest::TestDefaultIgnorableCallback() {
680 UErrorCode status = U_ZERO_ERROR;
681 const char *cnv_name = "euc-jp-2007";
682 const char *pattern_ignorable = "[:Default_Ignorable_Code_Point:]";
683 const char *pattern_not_ignorable =
684 "[[:^Default_Ignorable_Code_Point:]"
685 // For test performance, skip large ranges that will likely remain unassigned
686 // for a long time, and private use code points.
687 "-[\\U00040000-\\U000DFFFF]-[:Co:]"
688 "]";
689
690 LocalPointer<UnicodeSet> set_ignorable(new UnicodeSet(pattern_ignorable, status));
691 if (U_FAILURE(status)) {
692 dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_ignorable, u_errorName(status));
693 return;
694 }
695
696 LocalPointer<UnicodeSet> set_not_ignorable(new UnicodeSet(pattern_not_ignorable, status));
697 if (U_FAILURE(status)) {
698 dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_not_ignorable, u_errorName(status));
699 return;
700 }
701
702 LocalUConverterPointer cnv(cnv_open(cnv_name, status));
703 if (U_FAILURE(status)) {
704 dataerrln("Unable to open converter: %s - %s\n", cnv_name, u_errorName(status));
705 return;
706 }
707
708 // set callback for the converter
709 ucnv_setFromUCallBack(cnv.getAlias(), UCNV_FROM_U_CALLBACK_SUBSTITUTE, NULL, NULL, NULL, &status);
710
711 UChar32 input[1];
712 char output[10];
713 int32_t outputLength;
714
715 // test default ignorables are ignored
716 UnicodeSetIterator iter(*set_ignorable);
717 while (iter.next()) {
718 status = U_ZERO_ERROR;
719 outputLength= 0;
720
721 input[0] = iter.getCodepoint();
722
723 outputLength = ucnv_fromUChars(cnv.getAlias(), output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
724 if (U_FAILURE(status) || outputLength != 0) {
725 errln("Ignorable code point: U+%04X not skipped as expected - %s", input[0], u_errorName(status));
726 }
727 }
728
729 // test non-ignorables are not ignored
730 iter.reset(*set_not_ignorable);
731 while (iter.next()) {
732 status = U_ZERO_ERROR;
733 outputLength= 0;
734
735 input[0] = iter.getCodepoint();
736
737 if (input[0] == 0) {
738 continue;
739 }
740
741 outputLength = ucnv_fromUChars(cnv.getAlias(), output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
742 if (U_FAILURE(status) || outputLength <= 0) {
743 errln("Non-ignorable code point: U+%04X skipped unexpectedly - %s", input[0], u_errorName(status));
744 }
745 }
746 }
747
748 void
TestUTF8ToUTF8Overflow()749 ConversionTest::TestUTF8ToUTF8Overflow() {
750 IcuTestErrorCode errorCode(*this, "TestUTF8ToUTF8Overflow");
751 LocalUConverterPointer cnv1(ucnv_open("UTF-8", errorCode));
752 LocalUConverterPointer cnv2(ucnv_open("UTF-8", errorCode));
753 static const char *text = "aä"; // ä: 2 bytes
754 const char *source = text;
755 const char *sourceLimit = text + strlen(text);
756 char result[20];
757 char *target = result;
758 const char *targetLimit = result + sizeof(result);
759 UChar buffer16[20];
760 UChar *pivotSource = buffer16;
761 UChar *pivotTarget = buffer16;
762 const UChar *pivotLimit = buffer16 + UPRV_LENGTHOF(buffer16);
763 int32_t length;
764
765 // Convert with insufficient target capacity.
766 result[2] = 5;
767 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
768 &target, result + 2, &source, sourceLimit,
769 buffer16, &pivotSource, &pivotTarget, pivotLimit,
770 FALSE, FALSE, errorCode);
771 assertEquals("overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
772 length = (int32_t)(target - result);
773 assertEquals("number of bytes written", 2, length);
774 assertEquals("next byte not clobbered", 5, result[2]);
775
776 // Convert the rest and flush.
777 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
778 &target, targetLimit, &source, sourceLimit,
779 buffer16, &pivotSource, &pivotTarget, pivotLimit,
780 FALSE, TRUE, errorCode);
781
782 assertSuccess("UTF-8->UTF-8", errorCode);
783 length = (int32_t)(target - result);
784 assertEquals("3 bytes", 3, length);
785 if (length == 3) {
786 assertTrue("result same as input", memcmp(text, result, length) == 0);
787 }
788
789 ucnv_reset(cnv1.getAlias());
790 ucnv_reset(cnv2.getAlias());
791 memset(result, 0, sizeof(result));
792 static const char *text2 = "a��"; // U+1F6B2 bicycle: 4 bytes
793 source = text2;
794 sourceLimit = text2 + strlen(text2);
795 target = result;
796 pivotSource = pivotTarget = buffer16;
797
798 // Convert with insufficient target capacity.
799 result[3] = 5;
800 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
801 &target, result + 3, &source, sourceLimit,
802 buffer16, &pivotSource, &pivotTarget, pivotLimit,
803 FALSE, FALSE, errorCode);
804 assertEquals("text2 overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
805 length = (int32_t)(target - result);
806 assertEquals("text2 number of bytes written", 3, length);
807 assertEquals("text2 next byte not clobbered", 5, result[3]);
808
809 // Convert the rest and flush.
810 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
811 &target, targetLimit, &source, sourceLimit,
812 buffer16, &pivotSource, &pivotTarget, pivotLimit,
813 FALSE, TRUE, errorCode);
814
815 assertSuccess("text2 UTF-8->UTF-8", errorCode);
816 length = (int32_t)(target - result);
817 assertEquals("text2 5 bytes", 5, length);
818 if (length == 5) {
819 assertTrue("text2 result same as input", memcmp(text2, result, length) == 0);
820 }
821
822 ucnv_reset(cnv1.getAlias());
823 ucnv_reset(cnv2.getAlias());
824 memset(result, 0, sizeof(result));
825 static const char *illFormed = "\xf1\x91\x93\x96\x91\x94"; // U+514D6 + two more trail bytes
826 source = illFormed;
827 sourceLimit = illFormed + strlen(illFormed);
828 target = result;
829 pivotSource = pivotTarget = buffer16;
830
831 ucnv_setToUCallBack(cnv1.getAlias(), UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr, nullptr, errorCode);
832
833 // Convert only two bytes and flush (but expect failure).
834 char errorBytes[10];
835 int8_t errorLength;
836 result[0] = 5;
837 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
838 &target, targetLimit, &source, source + 2,
839 buffer16, &pivotSource, &pivotTarget, pivotLimit,
840 FALSE, TRUE, errorCode);
841 assertEquals("illFormed truncated", U_TRUNCATED_CHAR_FOUND, errorCode.reset());
842 length = (int32_t)(target - result);
843 assertEquals("illFormed number of bytes written", 0, length);
844 errorLength = UPRV_LENGTHOF(errorBytes);
845 ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
846 assertEquals("illFormed truncated errorLength", 2, (int32_t)errorLength);
847 if (errorLength == 2) {
848 assertEquals("illFormed truncated errorBytes", 0xf191,
849 ((int32_t)(uint8_t)errorBytes[0] << 8) | (uint8_t)errorBytes[1]);
850 }
851
852 // Continue conversion starting with a trail byte.
853 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
854 &target, targetLimit, &source, sourceLimit,
855 buffer16, &pivotSource, &pivotTarget, pivotLimit,
856 FALSE, TRUE, errorCode);
857
858 assertEquals("illFormed trail byte", U_ILLEGAL_CHAR_FOUND, errorCode.reset());
859 length = (int32_t)(target - result);
860 assertEquals("illFormed trail byte number of bytes written", 0, length);
861 errorLength = UPRV_LENGTHOF(errorBytes);
862 ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
863 assertEquals("illFormed trail byte errorLength", 1, (int32_t)errorLength);
864 if (errorLength == 1) {
865 assertEquals("illFormed trail byte errorBytes", 0x93, (int32_t)(uint8_t)errorBytes[0]);
866 }
867 }
868
869 void
TestUTF8ToUTF8Streaming()870 ConversionTest::TestUTF8ToUTF8Streaming() {
871 IcuTestErrorCode errorCode(*this, "TestUTF8ToUTF8Streaming");
872 LocalUConverterPointer cnv1(ucnv_open("UTF-8", errorCode));
873 LocalUConverterPointer cnv2(ucnv_open("UTF-8", errorCode));
874
875 // UTF8 encoded cyrillic part of 'Lorem ipsum'
876 static const char* text =
877 "\xd0\xb5\xd1\x82\x20\xd1\x81\xd1\x86\xd0\xb0\xd0\xb5\xd0\xb2\xd0"
878 "\xbe\xd0\xbb\xd0\xb0\x20\xd1\x81\xd0\xb0\xd0\xb4\xd0\xb8\xd0\xbf"
879 "\xd1\x81\xd1\x86\xd0\xb8\xd0\xbd\xd0\xb3\x20\xd0\xb0\xd1\x86\xd1"
880 "\x86\xd0\xbe\xd0\xbc\xd0\xbc\xd0\xbe\xd0\xb4\xd0\xb0\xd1\x80\xd0"
881 "\xb5\x20\xd1\x85\xd0\xb0\xd1\x81";
882
883 int32_t chunk1 = 25; // partial lead at the end: 0xd0
884 int32_t chunk2 = 47; // partial tail at the beginning: 0xb0
885
886 char result[128];
887
888 int32_t sourceLen = (int32_t)strlen(text);
889 const char* source = text;
890 const char* sourceLimit = text + chunk1;
891
892 int32_t targetLen = sizeof(result);
893 char* target = result;
894 const char* targetLimit = result + targetLen;
895
896 UChar buffer16[20];
897 UChar* pivotSource = buffer16;
898 UChar* pivotTarget = buffer16;
899 const UChar* pivotLimit = buffer16 + UPRV_LENGTHOF(buffer16);
900
901 int32_t length;
902 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
903 &target, result + targetLen, &source, sourceLimit,
904 buffer16, &pivotSource, &pivotTarget, pivotLimit,
905 FALSE, FALSE, errorCode);
906
907 length = (int32_t)(target - result);
908 targetLen -= length;
909 assertEquals("First chunk -1 doesn't match converted length", chunk1 - 1, length);
910
911 source = text + chunk1;
912 sourceLimit = source + chunk2;
913
914 // Convert the rest and flush.
915 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
916 &target, targetLimit, &source, sourceLimit,
917 buffer16, &pivotSource, &pivotTarget, pivotLimit,
918 FALSE, TRUE, errorCode);
919
920 length = (int32_t)(target - result - length);
921 targetLen -= length;
922 assertEquals("Second chunk + 2 doesn't match converted length", chunk2 + 1, length);
923
924 assertEquals("Full text length match", sourceLen, sizeof(result) - targetLen);
925 assertSuccess("UTF-8->UTF-8", errorCode);
926 }
927
928 // open testdata or ICU data converter ------------------------------------- ***
929
930 UConverter *
cnv_open(const char * name,UErrorCode & errorCode)931 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
932 if(name!=NULL && *name=='+') {
933 // Converter names that start with '+' are ignored in ICU4J tests.
934 ++name;
935 }
936 if(name!=NULL && *name=='*') {
937 /* loadTestData(): set the data directory */
938 return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
939 } else {
940 return ucnv_open(name, &errorCode);
941 }
942 }
943
944 // output helpers ---------------------------------------------------------- ***
945
946 static inline char
hexDigit(uint8_t digit)947 hexDigit(uint8_t digit) {
948 return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
949 }
950
951 static char *
printBytes(const uint8_t * bytes,int32_t length,char * out)952 printBytes(const uint8_t *bytes, int32_t length, char *out) {
953 uint8_t b;
954
955 if(length>0) {
956 b=*bytes++;
957 --length;
958 *out++=hexDigit((uint8_t)(b>>4));
959 *out++=hexDigit((uint8_t)(b&0xf));
960 }
961
962 while(length>0) {
963 b=*bytes++;
964 --length;
965 *out++=' ';
966 *out++=hexDigit((uint8_t)(b>>4));
967 *out++=hexDigit((uint8_t)(b&0xf));
968 }
969 *out++=0;
970 return out;
971 }
972
973 static char *
printUnicode(const UChar * unicode,int32_t length,char * out)974 printUnicode(const UChar *unicode, int32_t length, char *out) {
975 UChar32 c;
976 int32_t i;
977
978 for(i=0; i<length;) {
979 if(i>0) {
980 *out++=' ';
981 }
982 U16_NEXT(unicode, i, length, c);
983 // write 4..6 digits
984 if(c>=0x100000) {
985 *out++='1';
986 }
987 if(c>=0x10000) {
988 *out++=hexDigit((uint8_t)((c>>16)&0xf));
989 }
990 *out++=hexDigit((uint8_t)((c>>12)&0xf));
991 *out++=hexDigit((uint8_t)((c>>8)&0xf));
992 *out++=hexDigit((uint8_t)((c>>4)&0xf));
993 *out++=hexDigit((uint8_t)(c&0xf));
994 }
995 *out++=0;
996 return out;
997 }
998
999 static char *
printOffsets(const int32_t * offsets,int32_t length,char * out)1000 printOffsets(const int32_t *offsets, int32_t length, char *out) {
1001 int32_t i, o, d;
1002
1003 if(offsets==NULL) {
1004 length=0;
1005 }
1006
1007 for(i=0; i<length; ++i) {
1008 if(i>0) {
1009 *out++=' ';
1010 }
1011 o=offsets[i];
1012
1013 // print all offsets with 2 characters each (-x, -9..99, xx)
1014 if(o<-9) {
1015 *out++='-';
1016 *out++='x';
1017 } else if(o<0) {
1018 *out++='-';
1019 *out++=(char)('0'-o);
1020 } else if(o<=99) {
1021 *out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
1022 *out++=(char)('0'+o%10);
1023 } else /* o>99 */ {
1024 *out++='x';
1025 *out++='x';
1026 }
1027 }
1028 *out++=0;
1029 return out;
1030 }
1031
1032 // toUnicode test worker functions ----------------------------------------- ***
1033
1034 static int32_t
stepToUnicode(ConversionCase & cc,UConverter * cnv,UChar * result,int32_t resultCapacity,int32_t * resultOffsets,int32_t step,UErrorCode * pErrorCode)1035 stepToUnicode(ConversionCase &cc, UConverter *cnv,
1036 UChar *result, int32_t resultCapacity,
1037 int32_t *resultOffsets, /* also resultCapacity */
1038 int32_t step,
1039 UErrorCode *pErrorCode) {
1040 const char *source, *sourceLimit, *bytesLimit;
1041 UChar *target, *targetLimit, *resultLimit;
1042 UBool flush;
1043
1044 source=(const char *)cc.bytes;
1045 target=result;
1046 bytesLimit=source+cc.bytesLength;
1047 resultLimit=result+resultCapacity;
1048
1049 if(step>=0) {
1050 // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
1051 // move only one buffer (in vs. out) at a time to be extra mean
1052 // step==0 performs bulk conversion and generates offsets
1053
1054 // initialize the partial limits for the loop
1055 if(step==0) {
1056 // use the entire buffers
1057 sourceLimit=bytesLimit;
1058 targetLimit=resultLimit;
1059 flush=cc.finalFlush;
1060 } else {
1061 // start with empty partial buffers
1062 sourceLimit=source;
1063 targetLimit=target;
1064 flush=FALSE;
1065
1066 // output offsets only for bulk conversion
1067 resultOffsets=NULL;
1068 }
1069
1070 for(;;) {
1071 // resetting the opposite conversion direction must not affect this one
1072 ucnv_resetFromUnicode(cnv);
1073
1074 // convert
1075 ucnv_toUnicode(cnv,
1076 &target, targetLimit,
1077 &source, sourceLimit,
1078 resultOffsets,
1079 flush, pErrorCode);
1080
1081 // check pointers and errors
1082 if(source>sourceLimit || target>targetLimit) {
1083 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1084 break;
1085 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1086 if(target!=targetLimit) {
1087 // buffer overflow must only be set when the target is filled
1088 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1089 break;
1090 } else if(targetLimit==resultLimit) {
1091 // not just a partial overflow
1092 break;
1093 }
1094
1095 // the partial target is filled, set a new limit, reset the error and continue
1096 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1097 *pErrorCode=U_ZERO_ERROR;
1098 } else if(U_FAILURE(*pErrorCode)) {
1099 // some other error occurred, done
1100 break;
1101 } else {
1102 if(source!=sourceLimit) {
1103 // when no error occurs, then the input must be consumed
1104 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1105 break;
1106 }
1107
1108 if(sourceLimit==bytesLimit) {
1109 // we are done
1110 break;
1111 }
1112
1113 // the partial conversion succeeded, set a new limit and continue
1114 sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
1115 flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
1116 }
1117 }
1118 } else /* step<0 */ {
1119 /*
1120 * step==-1: call only ucnv_getNextUChar()
1121 * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
1122 * if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
1123 * else give it at most (-step-2)/2 bytes
1124 */
1125 UChar32 c;
1126
1127 // end the loop by getting an index out of bounds error
1128 for(;;) {
1129 // resetting the opposite conversion direction must not affect this one
1130 ucnv_resetFromUnicode(cnv);
1131
1132 // convert
1133 if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
1134 sourceLimit=source; // use sourceLimit not as a real limit
1135 // but to remember the pre-getNextUChar source pointer
1136 c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
1137
1138 // check pointers and errors
1139 if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
1140 if(source!=bytesLimit) {
1141 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1142 } else {
1143 *pErrorCode=U_ZERO_ERROR;
1144 }
1145 break;
1146 } else if(U_FAILURE(*pErrorCode)) {
1147 break;
1148 }
1149 // source may not move if c is from previous overflow
1150
1151 if(target==resultLimit) {
1152 *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1153 break;
1154 }
1155 if(c<=0xffff) {
1156 *target++=(UChar)c;
1157 } else {
1158 *target++=U16_LEAD(c);
1159 if(target==resultLimit) {
1160 *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1161 break;
1162 }
1163 *target++=U16_TRAIL(c);
1164 }
1165
1166 // alternate between -n-1 and -n but leave -1 alone
1167 if(step<-1) {
1168 ++step;
1169 }
1170 } else /* step is even */ {
1171 // allow only one UChar output
1172 targetLimit=target<resultLimit ? target+1 : resultLimit;
1173
1174 // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
1175 // and never output offsets
1176 if(step==-2) {
1177 sourceLimit=bytesLimit;
1178 } else {
1179 sourceLimit=source+(-step-2)/2;
1180 if(sourceLimit>bytesLimit) {
1181 sourceLimit=bytesLimit;
1182 }
1183 }
1184
1185 ucnv_toUnicode(cnv,
1186 &target, targetLimit,
1187 &source, sourceLimit,
1188 NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
1189
1190 // check pointers and errors
1191 if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1192 if(target!=targetLimit) {
1193 // buffer overflow must only be set when the target is filled
1194 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1195 break;
1196 } else if(targetLimit==resultLimit) {
1197 // not just a partial overflow
1198 break;
1199 }
1200
1201 // the partial target is filled, set a new limit and continue
1202 *pErrorCode=U_ZERO_ERROR;
1203 } else if(U_FAILURE(*pErrorCode)) {
1204 // some other error occurred, done
1205 break;
1206 } else {
1207 if(source!=sourceLimit) {
1208 // when no error occurs, then the input must be consumed
1209 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1210 break;
1211 }
1212
1213 // we are done (flush==TRUE) but we continue, to get the index out of bounds error above
1214 }
1215
1216 --step;
1217 }
1218 }
1219 }
1220
1221 return (int32_t)(target-result);
1222 }
1223
1224 UBool
ToUnicodeCase(ConversionCase & cc,UConverterToUCallback callback,const char * option)1225 ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
1226 // open the converter
1227 IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
1228 LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
1229 // with no data, the above crashes with "pointer being freed was not allocated" for charset "x11-compound-text", see #13078
1230 if(errorCode.isFailure()) {
1231 errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1232 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
1233 errorCode.reset();
1234 return FALSE;
1235 }
1236
1237 // set the callback
1238 if(callback!=NULL) {
1239 ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
1240 if(U_FAILURE(errorCode)) {
1241 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
1242 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1243 return FALSE;
1244 }
1245 }
1246
1247 int32_t resultOffsets[256];
1248 UChar result[256];
1249 int32_t resultLength;
1250 UBool ok;
1251
1252 static const struct {
1253 int32_t step;
1254 const char *name;
1255 } steps[]={
1256 { 0, "bulk" }, // must be first for offsets to be checked
1257 { 1, "step=1" },
1258 { 3, "step=3" },
1259 { 7, "step=7" },
1260 { -1, "getNext" },
1261 { -2, "toU(bulk)+getNext" },
1262 { -3, "getNext+toU(bulk)" },
1263 { -4, "toU(1)+getNext" },
1264 { -5, "getNext+toU(1)" },
1265 { -12, "toU(5)+getNext" },
1266 { -13, "getNext+toU(5)" },
1267 };
1268 int32_t i, step;
1269
1270 ok=TRUE;
1271 for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1272 step=steps[i].step;
1273 if(step<0 && !cc.finalFlush) {
1274 // skip ucnv_getNextUChar() if !finalFlush because
1275 // ucnv_getNextUChar() always implies flush
1276 continue;
1277 }
1278 if(step!=0) {
1279 // bulk test is first, then offsets are not checked any more
1280 cc.offsets=NULL;
1281 }
1282 else {
1283 for (int32_t i = 0; i < UPRV_LENGTHOF(resultOffsets); i++) {
1284 resultOffsets[i] = -1;
1285 }
1286 }
1287 for (int32_t i = 0; i < UPRV_LENGTHOF(result); i++) {
1288 result[i] = -1;
1289 }
1290 errorCode.reset();
1291 resultLength=stepToUnicode(cc, cnv.getAlias(),
1292 result, UPRV_LENGTHOF(result),
1293 step==0 ? resultOffsets : NULL,
1294 step, errorCode);
1295 ok=checkToUnicode(
1296 cc, cnv.getAlias(), steps[i].name,
1297 result, resultLength,
1298 cc.offsets!=NULL ? resultOffsets : NULL,
1299 errorCode);
1300 if(errorCode.isFailure() || !cc.finalFlush) {
1301 // reset if an error occurred or we did not flush
1302 // otherwise do nothing to make sure that flushing resets
1303 ucnv_resetToUnicode(cnv.getAlias());
1304 }
1305 if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
1306 errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1307 cc.caseNr, cc.charset, resultLength);
1308 }
1309 if (result[resultLength] != (UChar)-1) {
1310 errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
1311 cc.caseNr, cc.charset, resultLength);
1312 }
1313 }
1314
1315 // not a real loop, just a convenience for breaking out of the block
1316 while(ok && cc.finalFlush) {
1317 // test ucnv_toUChars()
1318 memset(result, 0, sizeof(result));
1319
1320 errorCode.reset();
1321 resultLength=ucnv_toUChars(cnv.getAlias(),
1322 result, UPRV_LENGTHOF(result),
1323 (const char *)cc.bytes, cc.bytesLength,
1324 errorCode);
1325 ok=checkToUnicode(
1326 cc, cnv.getAlias(), "toUChars",
1327 result, resultLength,
1328 NULL,
1329 errorCode);
1330 if(!ok) {
1331 break;
1332 }
1333
1334 // test preflighting
1335 // keep the correct result for simple checking
1336 errorCode.reset();
1337 resultLength=ucnv_toUChars(cnv.getAlias(),
1338 NULL, 0,
1339 (const char *)cc.bytes, cc.bytesLength,
1340 errorCode);
1341 if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
1342 errorCode.reset();
1343 }
1344 ok=checkToUnicode(
1345 cc, cnv.getAlias(), "preflight toUChars",
1346 result, resultLength,
1347 NULL,
1348 errorCode);
1349 break;
1350 }
1351
1352 errorCode.reset(); // all errors have already been reported
1353 return ok;
1354 }
1355
1356 UBool
checkToUnicode(ConversionCase & cc,UConverter * cnv,const char * name,const UChar * result,int32_t resultLength,const int32_t * resultOffsets,UErrorCode resultErrorCode)1357 ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1358 const UChar *result, int32_t resultLength,
1359 const int32_t *resultOffsets,
1360 UErrorCode resultErrorCode) {
1361 char resultInvalidChars[8];
1362 int8_t resultInvalidLength;
1363 UErrorCode errorCode;
1364
1365 const char *msg;
1366
1367 // reset the message; NULL will mean "ok"
1368 msg=NULL;
1369
1370 errorCode=U_ZERO_ERROR;
1371 resultInvalidLength=sizeof(resultInvalidChars);
1372 ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
1373 if(U_FAILURE(errorCode)) {
1374 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
1375 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1376 return FALSE;
1377 }
1378
1379 // check everything that might have gone wrong
1380 if(cc.unicodeLength!=resultLength) {
1381 msg="wrong result length";
1382 } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
1383 msg="wrong result string";
1384 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
1385 msg="wrong offsets";
1386 } else if(cc.outErrorCode!=resultErrorCode) {
1387 msg="wrong error code";
1388 } else if(cc.invalidLength!=resultInvalidLength) {
1389 msg="wrong length of last invalid input";
1390 } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
1391 msg="wrong last invalid input";
1392 }
1393
1394 if(msg==NULL) {
1395 return TRUE;
1396 } else {
1397 char buffer[2000]; // one buffer for all strings
1398 char *s, *bytesString, *unicodeString, *resultString,
1399 *offsetsString, *resultOffsetsString,
1400 *invalidCharsString, *resultInvalidCharsString;
1401
1402 bytesString=s=buffer;
1403 s=printBytes(cc.bytes, cc.bytesLength, bytesString);
1404 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
1405 s=printUnicode(result, resultLength, resultString=s);
1406 s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
1407 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1408 s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
1409 s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
1410
1411 if((s-buffer)>(int32_t)sizeof(buffer)) {
1412 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
1413 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1414 exit(1);
1415 }
1416
1417 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1418 " bytes <%s>[%d]\n"
1419 " expected <%s>[%d]\n"
1420 " result <%s>[%d]\n"
1421 " offsets <%s>\n"
1422 " result offsets <%s>\n"
1423 " error code expected %s got %s\n"
1424 " invalidChars expected <%s> got <%s>\n",
1425 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1426 bytesString, cc.bytesLength,
1427 unicodeString, cc.unicodeLength,
1428 resultString, resultLength,
1429 offsetsString,
1430 resultOffsetsString,
1431 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1432 invalidCharsString, resultInvalidCharsString);
1433
1434 return FALSE;
1435 }
1436 }
1437
1438 // fromUnicode test worker functions --------------------------------------- ***
1439
1440 static int32_t
stepFromUTF8(ConversionCase & cc,UConverter * utf8Cnv,UConverter * cnv,char * result,int32_t resultCapacity,int32_t step,UErrorCode * pErrorCode)1441 stepFromUTF8(ConversionCase &cc,
1442 UConverter *utf8Cnv, UConverter *cnv,
1443 char *result, int32_t resultCapacity,
1444 int32_t step,
1445 UErrorCode *pErrorCode) {
1446 const char *source, *sourceLimit, *utf8Limit;
1447 UChar pivotBuffer[32];
1448 UChar *pivotSource, *pivotTarget, *pivotLimit;
1449 char *target, *targetLimit, *resultLimit;
1450 UBool flush;
1451
1452 source=cc.utf8;
1453 pivotSource=pivotTarget=pivotBuffer;
1454 target=result;
1455 utf8Limit=source+cc.utf8Length;
1456 resultLimit=result+resultCapacity;
1457
1458 // call ucnv_convertEx() with in/out buffers no larger than (step) at a time
1459 // move only one buffer (in vs. out) at a time to be extra mean
1460 // step==0 performs bulk conversion
1461
1462 // initialize the partial limits for the loop
1463 if(step==0) {
1464 // use the entire buffers
1465 sourceLimit=utf8Limit;
1466 targetLimit=resultLimit;
1467 flush=cc.finalFlush;
1468
1469 pivotLimit=pivotBuffer+UPRV_LENGTHOF(pivotBuffer);
1470 } else {
1471 // start with empty partial buffers
1472 sourceLimit=source;
1473 targetLimit=target;
1474 flush=FALSE;
1475
1476 // empty pivot is not allowed, make it of length step
1477 pivotLimit=pivotBuffer+step;
1478 }
1479
1480 for(;;) {
1481 // resetting the opposite conversion direction must not affect this one
1482 ucnv_resetFromUnicode(utf8Cnv);
1483 ucnv_resetToUnicode(cnv);
1484
1485 // convert
1486 ucnv_convertEx(cnv, utf8Cnv,
1487 &target, targetLimit,
1488 &source, sourceLimit,
1489 pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
1490 FALSE, flush, pErrorCode);
1491
1492 // check pointers and errors
1493 if(source>sourceLimit || target>targetLimit) {
1494 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1495 break;
1496 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1497 if(target!=targetLimit) {
1498 // buffer overflow must only be set when the target is filled
1499 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1500 break;
1501 } else if(targetLimit==resultLimit) {
1502 // not just a partial overflow
1503 break;
1504 }
1505
1506 // the partial target is filled, set a new limit, reset the error and continue
1507 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1508 *pErrorCode=U_ZERO_ERROR;
1509 } else if(U_FAILURE(*pErrorCode)) {
1510 if(pivotSource==pivotBuffer) {
1511 // toUnicode error, should not occur
1512 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1513 break;
1514 } else {
1515 // fromUnicode error
1516 // some other error occurred, done
1517 break;
1518 }
1519 } else {
1520 if(source!=sourceLimit) {
1521 // when no error occurs, then the input must be consumed
1522 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1523 break;
1524 }
1525
1526 if(sourceLimit==utf8Limit) {
1527 // we are done
1528 if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
1529 // ucnv_convertEx() warns about not terminating the output
1530 // but ucnv_fromUnicode() does not and so
1531 // checkFromUnicode() does not expect it
1532 *pErrorCode=U_ZERO_ERROR;
1533 }
1534 break;
1535 }
1536
1537 // the partial conversion succeeded, set a new limit and continue
1538 sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
1539 flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
1540 }
1541 }
1542
1543 return (int32_t)(target-result);
1544 }
1545
1546 static int32_t
stepFromUnicode(ConversionCase & cc,UConverter * cnv,char * result,int32_t resultCapacity,int32_t * resultOffsets,int32_t step,UErrorCode * pErrorCode)1547 stepFromUnicode(ConversionCase &cc, UConverter *cnv,
1548 char *result, int32_t resultCapacity,
1549 int32_t *resultOffsets, /* also resultCapacity */
1550 int32_t step,
1551 UErrorCode *pErrorCode) {
1552 const UChar *source, *sourceLimit, *unicodeLimit;
1553 char *target, *targetLimit, *resultLimit;
1554 UBool flush;
1555
1556 source=cc.unicode;
1557 target=result;
1558 unicodeLimit=source+cc.unicodeLength;
1559 resultLimit=result+resultCapacity;
1560
1561 // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
1562 // move only one buffer (in vs. out) at a time to be extra mean
1563 // step==0 performs bulk conversion and generates offsets
1564
1565 // initialize the partial limits for the loop
1566 if(step==0) {
1567 // use the entire buffers
1568 sourceLimit=unicodeLimit;
1569 targetLimit=resultLimit;
1570 flush=cc.finalFlush;
1571 } else {
1572 // start with empty partial buffers
1573 sourceLimit=source;
1574 targetLimit=target;
1575 flush=FALSE;
1576
1577 // output offsets only for bulk conversion
1578 resultOffsets=NULL;
1579 }
1580
1581 for(;;) {
1582 // resetting the opposite conversion direction must not affect this one
1583 ucnv_resetToUnicode(cnv);
1584
1585 // convert
1586 ucnv_fromUnicode(cnv,
1587 &target, targetLimit,
1588 &source, sourceLimit,
1589 resultOffsets,
1590 flush, pErrorCode);
1591
1592 // check pointers and errors
1593 if(source>sourceLimit || target>targetLimit) {
1594 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1595 break;
1596 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1597 if(target!=targetLimit) {
1598 // buffer overflow must only be set when the target is filled
1599 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1600 break;
1601 } else if(targetLimit==resultLimit) {
1602 // not just a partial overflow
1603 break;
1604 }
1605
1606 // the partial target is filled, set a new limit, reset the error and continue
1607 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1608 *pErrorCode=U_ZERO_ERROR;
1609 } else if(U_FAILURE(*pErrorCode)) {
1610 // some other error occurred, done
1611 break;
1612 } else {
1613 if(source!=sourceLimit) {
1614 // when no error occurs, then the input must be consumed
1615 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1616 break;
1617 }
1618
1619 if(sourceLimit==unicodeLimit) {
1620 // we are done
1621 break;
1622 }
1623
1624 // the partial conversion succeeded, set a new limit and continue
1625 sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
1626 flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
1627 }
1628 }
1629
1630 return (int32_t)(target-result);
1631 }
1632
1633 UBool
FromUnicodeCase(ConversionCase & cc,UConverterFromUCallback callback,const char * option)1634 ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
1635 UConverter *cnv;
1636 UErrorCode errorCode;
1637
1638 // open the converter
1639 errorCode=U_ZERO_ERROR;
1640 cnv=cnv_open(cc.charset, errorCode);
1641 if(U_FAILURE(errorCode)) {
1642 errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1643 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1644 return FALSE;
1645 }
1646 ucnv_resetToUnicode(utf8Cnv);
1647
1648 // set the callback
1649 if(callback!=NULL) {
1650 ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
1651 if(U_FAILURE(errorCode)) {
1652 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
1653 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1654 ucnv_close(cnv);
1655 return FALSE;
1656 }
1657 }
1658
1659 // set the fallbacks flag
1660 // TODO change with Jitterbug 2401, then add a similar call for toUnicode too
1661 ucnv_setFallback(cnv, cc.fallbacks);
1662
1663 // set the subchar
1664 int32_t length;
1665
1666 if(cc.setSub>0) {
1667 length=(int32_t)strlen(cc.subchar);
1668 ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
1669 if(U_FAILURE(errorCode)) {
1670 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
1671 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1672 ucnv_close(cnv);
1673 return FALSE;
1674 }
1675 } else if(cc.setSub<0) {
1676 ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
1677 if(U_FAILURE(errorCode)) {
1678 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
1679 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1680 ucnv_close(cnv);
1681 return FALSE;
1682 }
1683 }
1684
1685 // convert unicode to utf8
1686 char utf8[256];
1687 cc.utf8=utf8;
1688 u_strToUTF8(utf8, UPRV_LENGTHOF(utf8), &cc.utf8Length,
1689 cc.unicode, cc.unicodeLength,
1690 &errorCode);
1691 if(U_FAILURE(errorCode)) {
1692 // skip UTF-8 testing of a string with an unpaired surrogate,
1693 // or of one that's too long
1694 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1695 cc.utf8Length=-1;
1696 }
1697
1698 int32_t resultOffsets[256];
1699 char result[256];
1700 int32_t resultLength;
1701 UBool ok;
1702
1703 static const struct {
1704 int32_t step;
1705 const char *name, *utf8Name;
1706 } steps[]={
1707 { 0, "bulk", "utf8" }, // must be first for offsets to be checked
1708 { 1, "step=1", "utf8 step=1" },
1709 { 3, "step=3", "utf8 step=3" },
1710 { 7, "step=7", "utf8 step=7" }
1711 };
1712 int32_t i, step;
1713
1714 ok=TRUE;
1715 for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1716 step=steps[i].step;
1717 for (int32_t i = 0; i < UPRV_LENGTHOF(resultOffsets); i++) {
1718 resultOffsets[i] = -1;
1719 }
1720 for (int32_t i = 0; i < UPRV_LENGTHOF(result); i++) {
1721 result[i] = -1;
1722 }
1723 errorCode=U_ZERO_ERROR;
1724 resultLength=stepFromUnicode(cc, cnv,
1725 result, UPRV_LENGTHOF(result),
1726 step==0 ? resultOffsets : NULL,
1727 step, &errorCode);
1728 ok=checkFromUnicode(
1729 cc, cnv, steps[i].name,
1730 (uint8_t *)result, resultLength,
1731 cc.offsets!=NULL ? resultOffsets : NULL,
1732 errorCode);
1733 if(U_FAILURE(errorCode) || !cc.finalFlush) {
1734 // reset if an error occurred or we did not flush
1735 // otherwise do nothing to make sure that flushing resets
1736 ucnv_resetFromUnicode(cnv);
1737 }
1738 if (resultOffsets[resultLength] != -1) {
1739 errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1740 cc.caseNr, cc.charset, resultLength);
1741 }
1742 if (result[resultLength] != (char)-1) {
1743 errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
1744 cc.caseNr, cc.charset, resultLength);
1745 }
1746
1747 // bulk test is first, then offsets are not checked any more
1748 cc.offsets=NULL;
1749
1750 // test direct conversion from UTF-8
1751 if(cc.utf8Length>=0) {
1752 errorCode=U_ZERO_ERROR;
1753 resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
1754 result, UPRV_LENGTHOF(result),
1755 step, &errorCode);
1756 ok=checkFromUnicode(
1757 cc, cnv, steps[i].utf8Name,
1758 (uint8_t *)result, resultLength,
1759 NULL,
1760 errorCode);
1761 if(U_FAILURE(errorCode) || !cc.finalFlush) {
1762 // reset if an error occurred or we did not flush
1763 // otherwise do nothing to make sure that flushing resets
1764 ucnv_resetToUnicode(utf8Cnv);
1765 ucnv_resetFromUnicode(cnv);
1766 }
1767 }
1768 }
1769
1770 // not a real loop, just a convenience for breaking out of the block
1771 while(ok && cc.finalFlush) {
1772 // test ucnv_fromUChars()
1773 memset(result, 0, sizeof(result));
1774
1775 errorCode=U_ZERO_ERROR;
1776 resultLength=ucnv_fromUChars(cnv,
1777 result, UPRV_LENGTHOF(result),
1778 cc.unicode, cc.unicodeLength,
1779 &errorCode);
1780 ok=checkFromUnicode(
1781 cc, cnv, "fromUChars",
1782 (uint8_t *)result, resultLength,
1783 NULL,
1784 errorCode);
1785 if(!ok) {
1786 break;
1787 }
1788
1789 // test preflighting
1790 // keep the correct result for simple checking
1791 errorCode=U_ZERO_ERROR;
1792 resultLength=ucnv_fromUChars(cnv,
1793 NULL, 0,
1794 cc.unicode, cc.unicodeLength,
1795 &errorCode);
1796 if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
1797 errorCode=U_ZERO_ERROR;
1798 }
1799 ok=checkFromUnicode(
1800 cc, cnv, "preflight fromUChars",
1801 (uint8_t *)result, resultLength,
1802 NULL,
1803 errorCode);
1804 break;
1805 }
1806
1807 ucnv_close(cnv);
1808 return ok;
1809 }
1810
1811 UBool
checkFromUnicode(ConversionCase & cc,UConverter * cnv,const char * name,const uint8_t * result,int32_t resultLength,const int32_t * resultOffsets,UErrorCode resultErrorCode)1812 ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1813 const uint8_t *result, int32_t resultLength,
1814 const int32_t *resultOffsets,
1815 UErrorCode resultErrorCode) {
1816 UChar resultInvalidUChars[8];
1817 int8_t resultInvalidLength;
1818 UErrorCode errorCode;
1819
1820 const char *msg;
1821
1822 // reset the message; NULL will mean "ok"
1823 msg=NULL;
1824
1825 errorCode=U_ZERO_ERROR;
1826 resultInvalidLength=UPRV_LENGTHOF(resultInvalidUChars);
1827 ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
1828 if(U_FAILURE(errorCode)) {
1829 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
1830 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1831 return FALSE;
1832 }
1833
1834 // check everything that might have gone wrong
1835 if(cc.bytesLength!=resultLength) {
1836 msg="wrong result length";
1837 } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
1838 msg="wrong result string";
1839 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
1840 msg="wrong offsets";
1841 } else if(cc.outErrorCode!=resultErrorCode) {
1842 msg="wrong error code";
1843 } else if(cc.invalidLength!=resultInvalidLength) {
1844 msg="wrong length of last invalid input";
1845 } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
1846 msg="wrong last invalid input";
1847 }
1848
1849 if(msg==NULL) {
1850 return TRUE;
1851 } else {
1852 char buffer[2000]; // one buffer for all strings
1853 char *s, *unicodeString, *bytesString, *resultString,
1854 *offsetsString, *resultOffsetsString,
1855 *invalidCharsString, *resultInvalidUCharsString;
1856
1857 unicodeString=s=buffer;
1858 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
1859 s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
1860 s=printBytes(result, resultLength, resultString=s);
1861 s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
1862 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1863 s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
1864 s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
1865
1866 if((s-buffer)>(int32_t)sizeof(buffer)) {
1867 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
1868 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1869 exit(1);
1870 }
1871
1872 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1873 " unicode <%s>[%d]\n"
1874 " expected <%s>[%d]\n"
1875 " result <%s>[%d]\n"
1876 " offsets <%s>\n"
1877 " result offsets <%s>\n"
1878 " error code expected %s got %s\n"
1879 " invalidChars expected <%s> got <%s>\n",
1880 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1881 unicodeString, cc.unicodeLength,
1882 bytesString, cc.bytesLength,
1883 resultString, resultLength,
1884 offsetsString,
1885 resultOffsetsString,
1886 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1887 invalidCharsString, resultInvalidUCharsString);
1888
1889 return FALSE;
1890 }
1891 }
1892
1893 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
1894