// © 2016 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html /******************************************************************** * COPYRIGHT: * Copyright (c) 1999-2015, International Business Machines Corporation and * others. All Rights Reserved. ********************************************************************/ #include "simplethread.h" #include "unicode/utypes.h" #include "unicode/ustring.h" #include "umutex.h" #include "cmemory.h" #include "cstring.h" #include "indiancal.h" #include "uparse.h" #include "unicode/localpointer.h" #include "unicode/resbund.h" #include "unicode/udata.h" #include "unicode/uloc.h" #include "unicode/locid.h" #include "putilimp.h" #include "intltest.h" #include "tsmthred.h" #include "unicode/ushape.h" #include "unicode/translit.h" #include "sharedobject.h" #include "unifiedcache.h" #include "uassert.h" #define TSMTHREAD_FAIL(msg) errln("%s at file %s, line %d", msg, __FILE__, __LINE__) #define TSMTHREAD_ASSERT(expr) {if (!(expr)) {TSMTHREAD_FAIL("Fail");}} #define TSMTHREAD_ASSERT_SUCCESS(status) {if (U_FAILURE(status)) { \ errln("file: %s:%d status = %s\n", __FILE__, __LINE__, u_errorName(status));}} MultithreadTest::MultithreadTest() { } MultithreadTest::~MultithreadTest() { } #include #include #include // tolower, toupper #include #include "unicode/putil.h" // for mthreadtest #include "unicode/numfmt.h" #include "unicode/choicfmt.h" #include "unicode/msgfmt.h" #include "unicode/locid.h" #include "unicode/coll.h" #include "unicode/calendar.h" #include "ucaconf.h" void MultithreadTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ ) { if (exec) logln("TestSuite MultithreadTest: "); TESTCASE_AUTO_BEGIN; TESTCASE_AUTO(TestThreads); TESTCASE_AUTO(TestMutex); #if !UCONFIG_NO_FORMATTING TESTCASE_AUTO(TestThreadedIntl); #endif #if !UCONFIG_NO_COLLATION TESTCASE_AUTO(TestCollators); #endif /* #if !UCONFIG_NO_COLLATION */ TESTCASE_AUTO(TestString); TESTCASE_AUTO(TestArabicShapingThreads); TESTCASE_AUTO(TestAnyTranslit); TESTCASE_AUTO(TestConditionVariables); TESTCASE_AUTO(TestUnifiedCache); #if !UCONFIG_NO_TRANSLITERATION TESTCASE_AUTO(TestBreakTranslit); TESTCASE_AUTO(TestIncDec); #if !UCONFIG_NO_FORMATTING TESTCASE_AUTO(Test20104); #endif /* #if !UCONFIG_NO_FORMATTING */ #endif /* #if !UCONFIG_NO_TRANSLITERATION */ TESTCASE_AUTO_END } //----------------------------------------------------------------------------------- // // TestThreads -- see if threads really work at all. // // Set up N threads pointing at N chars. When they are started, they will // set their chars. At the end we make sure they are all set. // //----------------------------------------------------------------------------------- class TestThreadsThread : public SimpleThread { public: TestThreadsThread(char* whatToChange) { fWhatToChange = whatToChange; } virtual void run() { Mutex m; *fWhatToChange = '*'; } private: char *fWhatToChange; }; void MultithreadTest::TestThreads() { static const int32_t THREADTEST_NRTHREADS = 8; char threadTestChars[THREADTEST_NRTHREADS + 1]; SimpleThread *threads[THREADTEST_NRTHREADS]; int32_t numThreadsStarted = 0; int32_t i; for(i=0;i" + UnicodeString(threadTestChars) + "<- Firing off threads.. "); for(i=0;istart() != 0) { errln("Error starting thread %d", i); } else { numThreadsStarted++; } logln(" Subthread started."); } if (numThreadsStarted != THREADTEST_NRTHREADS) { errln("Not all threads could be started for testing!"); return; } logln("Waiting for threads to be set.."); for(i=0; ijoin(); if (threadTestChars[i] != '*') { errln("%s:%d Thread %d failed.", __FILE__, __LINE__, i); } delete threads[i]; } } //----------------------------------------------------------------------------------- // // TestArabicShapeThreads -- see if calls to u_shapeArabic in many threads works successfully // // Set up N threads pointing at N chars. When they are started, they will make calls to doTailTest which tests // u_shapeArabic, if the calls are successful it will the set * chars. // At the end we make sure all threads managed to run u_shapeArabic successfully. // This is a unit test for ticket 9473 // //----------------------------------------------------------------------------------- class TestArabicShapeThreads : public SimpleThread { public: TestArabicShapeThreads() {}; virtual void run() { doTailTest(); }; private: void doTailTest(); }; void TestArabicShapeThreads::doTailTest(void) { static const UChar src[] = { 0x0020, 0x0633, 0 }; static const UChar dst_old[] = { 0xFEB1, 0x200B,0 }; static const UChar dst_new[] = { 0xFEB1, 0xFE73,0 }; UChar dst[3] = { 0x0000, 0x0000,0 }; int32_t length; UErrorCode status; for (int32_t loopCount = 0; loopCount < 100; loopCount++) { status = U_ZERO_ERROR; length = u_shapeArabic(src, -1, dst, UPRV_LENGTHOF(dst), U_SHAPE_LETTERS_SHAPE|U_SHAPE_SEEN_TWOCELL_NEAR, &status); if(U_FAILURE(status)) { IntlTest::gTest->errln("Fail: status %s\n", u_errorName(status)); return; } else if(length!=2) { IntlTest::gTest->errln("Fail: len %d expected 3\n", length); return; } else if(u_strncmp(dst,dst_old,UPRV_LENGTHOF(dst))) { IntlTest::gTest->errln("Fail: got U+%04X U+%04X expected U+%04X U+%04X\n", dst[0],dst[1],dst_old[0],dst_old[1]); return; } //"Trying new tail status = U_ZERO_ERROR; length = u_shapeArabic(src, -1, dst, UPRV_LENGTHOF(dst), U_SHAPE_LETTERS_SHAPE|U_SHAPE_SEEN_TWOCELL_NEAR|U_SHAPE_TAIL_NEW_UNICODE, &status); if(U_FAILURE(status)) { IntlTest::gTest->errln("Fail: status %s\n", u_errorName(status)); return; } else if(length!=2) { IntlTest::gTest->errln("Fail: len %d expected 3\n", length); return; } else if(u_strncmp(dst,dst_new,UPRV_LENGTHOF(dst))) { IntlTest::gTest->errln("Fail: got U+%04X U+%04X expected U+%04X U+%04X\n", dst[0],dst[1],dst_new[0],dst_new[1]); return; } } return; } void MultithreadTest::TestArabicShapingThreads() { TestArabicShapeThreads threads[30]; int32_t i; logln("-> do TestArabicShapingThreads <- Firing off threads.. "); for(i=0; i < UPRV_LENGTHOF(threads); i++) { if (threads[i].start() != 0) { errln("Error starting thread %d", i); } } for(i=0; i < UPRV_LENGTHOF(threads); i++) { threads[i].join(); } logln("->TestArabicShapingThreads <- Got all threads! cya"); } //----------------------------------------------------------------------- // // TestMutex - a simple (non-stress) test to verify that ICU mutexes // and condition variables are functioning. Does not test the use of // mutexes within ICU services, but rather that the // platform's mutex support is at least superficially there. // //---------------------------------------------------------------------- static UMutex gTestMutexA = U_MUTEX_INITIALIZER; static UConditionVar gThreadsCountChanged = U_CONDITION_INITIALIZER; static int gThreadsStarted = 0; static int gThreadsInMiddle = 0; static int gThreadsDone = 0; static const int TESTMUTEX_THREAD_COUNT = 40; class TestMutexThread : public SimpleThread { public: virtual void run() { // This is the code that each of the spawned threads runs. // All threads move together throught the started - middle - done sequence together, // waiting for all other threads to reach each point before advancing. umtx_lock(&gTestMutexA); gThreadsStarted += 1; umtx_condBroadcast(&gThreadsCountChanged); while (gThreadsStarted < TESTMUTEX_THREAD_COUNT) { if (gThreadsInMiddle != 0) { IntlTest::gTest->errln( "%s:%d gThreadsInMiddle = %d. Expected 0.", __FILE__, __LINE__, gThreadsInMiddle); return; } umtx_condWait(&gThreadsCountChanged, &gTestMutexA); } gThreadsInMiddle += 1; umtx_condBroadcast(&gThreadsCountChanged); while (gThreadsInMiddle < TESTMUTEX_THREAD_COUNT) { if (gThreadsDone != 0) { IntlTest::gTest->errln( "%s:%d gThreadsDone = %d. Expected 0.", __FILE__, __LINE__, gThreadsDone); return; } umtx_condWait(&gThreadsCountChanged, &gTestMutexA); } gThreadsDone += 1; umtx_condBroadcast(&gThreadsCountChanged); while (gThreadsDone < TESTMUTEX_THREAD_COUNT) { umtx_condWait(&gThreadsCountChanged, &gTestMutexA); } umtx_unlock(&gTestMutexA); } }; void MultithreadTest::TestMutex() { gThreadsStarted = 0; gThreadsInMiddle = 0; gThreadsDone = 0; int32_t i = 0; TestMutexThread threads[TESTMUTEX_THREAD_COUNT]; umtx_lock(&gTestMutexA); for (i=0; isetLocale(theLocale); fmt->applyPattern(pattern, realStatus); if (U_FAILURE(realStatus)) { delete fmt; return; } FieldPosition ignore = 0; fmt->format(myArgs,4,result,ignore,realStatus); delete fmt; } /** * Shared formatters & data used by instances of ThreadSafeFormat. * Exactly one instance of this class is created, and it is then shared concurrently * by the multiple instances of ThreadSafeFormat. */ class ThreadSafeFormatSharedData { public: ThreadSafeFormatSharedData(UErrorCode &status); ~ThreadSafeFormatSharedData(); LocalPointer fFormat; Formattable fYDDThing; Formattable fBBDThing; UnicodeString fYDDStr; UnicodeString fBBDStr; }; const ThreadSafeFormatSharedData *gSharedData = NULL; ThreadSafeFormatSharedData::ThreadSafeFormatSharedData(UErrorCode &status) { fFormat.adoptInstead(NumberFormat::createCurrencyInstance(Locale::getUS(), status)); static const UChar kYDD[] = { 0x59, 0x44, 0x44, 0x00 }; static const UChar kBBD[] = { 0x42, 0x42, 0x44, 0x00 }; fYDDThing.adoptObject(new CurrencyAmount(123.456, kYDD, status)); fBBDThing.adoptObject(new CurrencyAmount(987.654, kBBD, status)); if (U_FAILURE(status)) { return; } fFormat->format(fYDDThing, fYDDStr, NULL, status); fFormat->format(fBBDThing, fBBDStr, NULL, status); gSharedData = this; } ThreadSafeFormatSharedData::~ThreadSafeFormatSharedData() { gSharedData = NULL; } /** * Class for thread-safe testing of format. * Instances of this class appear as members of class FormatThreadTest. * Multiple instances of FormatThreadTest coexist. * ThreadSafeFormat::doStuff() is called concurrently to test the thread safety of * various shared format operations. */ class ThreadSafeFormat { public: /* give a unique offset to each thread */ ThreadSafeFormat(UErrorCode &status); UBool doStuff(int32_t offset, UnicodeString &appendErr, UErrorCode &status) const; private: LocalPointer fFormat; // formatter - en_US constructed currency }; ThreadSafeFormat::ThreadSafeFormat(UErrorCode &status) { fFormat.adoptInstead(NumberFormat::createCurrencyInstance(Locale::getUS(), status)); } static const UChar kUSD[] = { 0x55, 0x53, 0x44, 0x00 }; UBool ThreadSafeFormat::doStuff(int32_t offset, UnicodeString &appendErr, UErrorCode &status) const { UBool okay = TRUE; if(u_strcmp(fFormat->getCurrency(), kUSD)) { appendErr.append("fFormat currency != ") .append(kUSD) .append(", =") .append(fFormat->getCurrency()) .append("! "); okay = FALSE; } if(u_strcmp(gSharedData->fFormat->getCurrency(), kUSD)) { appendErr.append("gFormat currency != ") .append(kUSD) .append(", =") .append(gSharedData->fFormat->getCurrency()) .append("! "); okay = FALSE; } UnicodeString str; const UnicodeString *o=NULL; Formattable f; const NumberFormat *nf = NULL; // only operate on it as const. switch(offset%4) { case 0: f = gSharedData->fYDDThing; o = &gSharedData->fYDDStr; nf = gSharedData->fFormat.getAlias(); break; case 1: f = gSharedData->fBBDThing; o = &gSharedData->fBBDStr; nf = gSharedData->fFormat.getAlias(); break; case 2: f = gSharedData->fYDDThing; o = &gSharedData->fYDDStr; nf = fFormat.getAlias(); break; case 3: f = gSharedData->fBBDThing; o = &gSharedData->fBBDStr; nf = fFormat.getAlias(); break; } nf->format(f, str, NULL, status); if(*o != str) { appendErr.append(showDifference(*o, str)); okay = FALSE; } return okay; } UBool U_CALLCONV isAcceptable(void *, const char *, const char *, const UDataInfo *) { return TRUE; } //static UMTX debugMutex = NULL; //static UMTX gDebugMutex; class FormatThreadTest : public SimpleThread { public: int fNum; int fTraceInfo; LocalPointer fTSF; FormatThreadTest() // constructor is NOT multithread safe. : SimpleThread(), fNum(0), fTraceInfo(0), fTSF(NULL), fOffset(0) // the locale to use { UErrorCode status = U_ZERO_ERROR; // TODO: rearrange code to allow checking of status. fTSF.adoptInstead(new ThreadSafeFormat(status)); static int32_t fgOffset = 0; fgOffset += 3; fOffset = fgOffset; } virtual void run() { fTraceInfo = 1; LocalPointer percentFormatter; UErrorCode status = U_ZERO_ERROR; #if 0 // debugging code, for (int i=0; i<4000; i++) { status = U_ZERO_ERROR; UDataMemory *data1 = udata_openChoice(0, "res", "en_US", isAcceptable, 0, &status); UDataMemory *data2 = udata_openChoice(0, "res", "fr", isAcceptable, 0, &status); udata_close(data1); udata_close(data2); if (U_FAILURE(status)) { error("udata_openChoice failed.\n"); break; } } return; #endif #if 0 // debugging code, int m; for (m=0; m<4000; m++) { status = U_ZERO_ERROR; UResourceBundle *res = NULL; const char *localeName = NULL; Locale loc = Locale::getEnglish(); localeName = loc.getName(); // localeName = "en"; // ResourceBundle bund = ResourceBundle(0, loc, status); //umtx_lock(&gDebugMutex); res = ures_open(NULL, localeName, &status); //umtx_unlock(&gDebugMutex); //umtx_lock(&gDebugMutex); ures_close(res); //umtx_unlock(&gDebugMutex); if (U_FAILURE(status)) { error("Resource bundle construction failed.\n"); break; } } return; #endif // Keep this data here to avoid static initialization. FormatThreadTestData kNumberFormatTestData[] = { FormatThreadTestData((double)5.0, UnicodeString(u"5")), FormatThreadTestData( 6.0, UnicodeString(u"6")), FormatThreadTestData( 20.0, UnicodeString(u"20")), FormatThreadTestData( 8.0, UnicodeString(u"8")), FormatThreadTestData( 8.3, UnicodeString(u"8.3")), FormatThreadTestData( 12345, UnicodeString(u"12,345")), FormatThreadTestData( 81890.23, UnicodeString(u"81,890.23")), }; int32_t kNumberFormatTestDataLength = UPRV_LENGTHOF(kNumberFormatTestData); // Keep this data here to avoid static initialization. FormatThreadTestData kPercentFormatTestData[] = { FormatThreadTestData((double)5.0, CharsToUnicodeString("500\\u00a0%")), FormatThreadTestData( 1.0, CharsToUnicodeString("100\\u00a0%")), FormatThreadTestData( 0.26, CharsToUnicodeString("26\\u00a0%")), FormatThreadTestData( 16384.99, CharsToUnicodeString("1\\u202F638\\u202F499\\u00a0%")), // U+202F = NNBSP FormatThreadTestData( 81890.23, CharsToUnicodeString("8\\u202F189\\u202F023\\u00a0%")), }; int32_t kPercentFormatTestDataLength = UPRV_LENGTHOF(kPercentFormatTestData); int32_t iteration; status = U_ZERO_ERROR; LocalPointer formatter(NumberFormat::createInstance(Locale::getEnglish(),status)); if(U_FAILURE(status)) { IntlTest::gTest->dataerrln("%s:%d Error %s on NumberFormat::createInstance().", __FILE__, __LINE__, u_errorName(status)); goto cleanupAndReturn; } percentFormatter.adoptInstead(NumberFormat::createPercentInstance(Locale::getFrench(),status)); if(U_FAILURE(status)) { IntlTest::gTest->errln("%s:%d Error %s on NumberFormat::createPercentInstance().", __FILE__, __LINE__, u_errorName(status)); goto cleanupAndReturn; } for(iteration = 0;!IntlTest::gTest->getErrors() && iterationformat(kNumberFormatTestData[whichLine].number, output); if(0 != output.compare(kNumberFormatTestData[whichLine].string)) { IntlTest::gTest->errln("format().. expected " + kNumberFormatTestData[whichLine].string + " got " + output); goto cleanupAndReturn; } // Now check percent. output.remove(); whichLine = (iteration + fOffset)%kPercentFormatTestDataLength; percentFormatter->format(kPercentFormatTestData[whichLine].number, output); if(0 != output.compare(kPercentFormatTestData[whichLine].string)) { IntlTest::gTest->errln("percent format().. \n" + showDifference(kPercentFormatTestData[whichLine].string,output)); goto cleanupAndReturn; } // Test message error const int kNumberOfMessageTests = 3; UErrorCode statusToCheck; UnicodeString patternToCheck; Locale messageLocale; Locale countryToCheck; double currencyToCheck; UnicodeString expected; // load the cases. switch((iteration+fOffset) % kNumberOfMessageTests) { default: case 0: statusToCheck= U_FILE_ACCESS_ERROR; patternToCheck= "0:Someone from {2} is receiving a #{0}" " error - {1}. Their telephone call is costing " "{3,number,currency}."; // number,currency messageLocale= Locale("en","US"); countryToCheck= Locale("","HR"); currencyToCheck= 8192.77; expected= "0:Someone from Croatia is receiving a #4 error - " "U_FILE_ACCESS_ERROR. Their telephone call is costing $8,192.77."; break; case 1: statusToCheck= U_INDEX_OUTOFBOUNDS_ERROR; patternToCheck= "1:A customer in {2} is receiving a #{0} error - {1}. " "Their telephone call is costing {3,number,currency}."; // number,currency messageLocale= Locale("de","DE@currency=DEM"); countryToCheck= Locale("","BF"); currencyToCheck= 2.32; expected= CharsToUnicodeString( "1:A customer in Burkina Faso is receiving a #8 error - U_INDEX_OUTOFBOUNDS_ERROR. " "Their telephone call is costing 2,32\\u00A0DM."); break; case 2: statusToCheck= U_MEMORY_ALLOCATION_ERROR; patternToCheck= "2:user in {2} is receiving a #{0} error - {1}. " "They insist they just spent {3,number,currency} " "on memory."; // number,currency messageLocale= Locale("de","AT@currency=ATS"); // Austrian German countryToCheck= Locale("","US"); // hmm currencyToCheck= 40193.12; expected= CharsToUnicodeString( "2:user in Vereinigte Staaten is receiving a #7 error" " - U_MEMORY_ALLOCATION_ERROR. They insist they just spent" " \\u00f6S\\u00A040.193,12 on memory."); break; } UnicodeString result; UErrorCode status = U_ZERO_ERROR; formatErrorMessage(status,patternToCheck,messageLocale,statusToCheck, countryToCheck,currencyToCheck,result); if(U_FAILURE(status)) { UnicodeString tmp(u_errorName(status)); IntlTest::gTest->errln("Failure on message format, pattern=" + patternToCheck + ", error = " + tmp); goto cleanupAndReturn; } if(result != expected) { IntlTest::gTest->errln("PatternFormat: \n" + showDifference(expected,result)); goto cleanupAndReturn; } // test the Thread Safe Format UnicodeString appendErr; if(!fTSF->doStuff(fNum, appendErr, status)) { IntlTest::gTest->errln(appendErr); goto cleanupAndReturn; } } /* end of for loop */ cleanupAndReturn: fTraceInfo = 2; } private: int32_t fOffset; // where we are testing from. }; // ** The actual test function. void MultithreadTest::TestThreadedIntl() { UnicodeString theErr; UErrorCode threadSafeErr = U_ZERO_ERROR; ThreadSafeFormatSharedData sharedData(threadSafeErr); assertSuccess("initializing ThreadSafeFormat", threadSafeErr, TRUE); // // Create and start the test threads // logln("Spawning: %d threads * %d iterations each.", kFormatThreadThreads, kFormatThreadIterations); FormatThreadTest tests[kFormatThreadThreads]; int32_t j; for(j = 0; j < UPRV_LENGTHOF(tests); j++) { tests[j].fNum = j; int32_t threadStatus = tests[j].start(); if (threadStatus != 0) { errln("%s:%d System Error %d starting thread number %d.", __FILE__, __LINE__, threadStatus, j); return; } } for (j=0; j= 3 && (s[0] == 0xfb2 || s[0] == 0xfb3) && s[1] == 0x334 && (s[2] == 0xf73 || s[2] == 0xf75 || s[2] == 0xf81)) { return TRUE; } return FALSE; } static UCollationResult normalizeResult(int32_t result) { return result<0 ? UCOL_LESS : result==0 ? UCOL_EQUAL : UCOL_GREATER; } class CollatorThreadTest : public SimpleThread { private: const Collator *coll; const Line *lines; int32_t noLines; UBool isAtLeastUCA62; public: CollatorThreadTest() : SimpleThread(), coll(NULL), lines(NULL), noLines(0), isAtLeastUCA62(TRUE) { }; void setCollator(Collator *c, Line *l, int32_t nl, UBool atLeastUCA62) { coll = c; lines = l; noLines = nl; isAtLeastUCA62 = atLeastUCA62; } virtual void run() { uint8_t sk1[1024], sk2[1024]; uint8_t *oldSk = NULL, *newSk = sk1; int32_t oldLen = 0; int32_t prev = 0; int32_t i = 0; for(i = 0; i < noLines; i++) { if(lines[i].buflen == 0) { continue; } if(skipLineBecauseOfBug(lines[i].buff, lines[i].buflen)) { continue; } int32_t resLen = coll->getSortKey(lines[i].buff, lines[i].buflen, newSk, 1024); if(oldSk != NULL) { int32_t skres = strcmp((char *)oldSk, (char *)newSk); int32_t cmpres = coll->compare(lines[prev].buff, lines[prev].buflen, lines[i].buff, lines[i].buflen); int32_t cmpres2 = coll->compare(lines[i].buff, lines[i].buflen, lines[prev].buff, lines[prev].buflen); if(cmpres != -cmpres2) { IntlTest::gTest->errln(UnicodeString("Compare result not symmetrical on line ") + (i + 1)); break; } if(cmpres != normalizeResult(skres)) { IntlTest::gTest->errln(UnicodeString("Difference between coll->compare and sortkey compare on line ") + (i + 1)); break; } int32_t res = cmpres; if(res == 0 && !isAtLeastUCA62) { // Up to UCA 6.1, the collation test files use a custom tie-breaker, // comparing the raw input strings. res = u_strcmpCodePointOrder(lines[prev].buff, lines[i].buff); // Starting with UCA 6.2, the collation test files use the standard UCA tie-breaker, // comparing the NFD versions of the input strings, // which we do via setting strength=identical. } if(res > 0) { IntlTest::gTest->errln(UnicodeString("Line is not greater or equal than previous line, for line ") + (i + 1)); break; } } oldSk = newSk; oldLen = resLen; (void)oldLen; // Suppress set but not used warning. prev = i; newSk = (newSk == sk1)?sk2:sk1; } } }; void MultithreadTest::TestCollators() { UErrorCode status = U_ZERO_ERROR; FILE *testFile = NULL; char testDataPath[1024]; strcpy(testDataPath, IntlTest::getSourceTestData(status)); if (U_FAILURE(status)) { errln("ERROR: could not open test data %s", u_errorName(status)); return; } strcat(testDataPath, "CollationTest_"); const char* type = "NON_IGNORABLE"; const char *ext = ".txt"; if(testFile) { fclose(testFile); } char buffer[1024]; strcpy(buffer, testDataPath); strcat(buffer, type); size_t bufLen = strlen(buffer); // we try to open 3 files: // path/CollationTest_type.txt // path/CollationTest_type_SHORT.txt // path/CollationTest_type_STUB.txt // we are going to test with the first one that we manage to open. strcpy(buffer+bufLen, ext); testFile = fopen(buffer, "rb"); if(testFile == 0) { strcpy(buffer+bufLen, "_SHORT"); strcat(buffer, ext); testFile = fopen(buffer, "rb"); if(testFile == 0) { strcpy(buffer+bufLen, "_STUB"); strcat(buffer, ext); testFile = fopen(buffer, "rb"); if (testFile == 0) { *(buffer+bufLen) = 0; dataerrln("could not open any of the conformance test files, tried opening base %s", buffer); return; } else { infoln( "INFO: Working with the stub file.\n" "If you need the full conformance test, please\n" "download the appropriate data files from:\n" "http://source.icu-project.org/repos/icu/tools/trunk/unicodetools/com/ibm/text/data/"); } } } LocalArray lines(new Line[200000]); memset(lines.getAlias(), 0, sizeof(Line)*200000); int32_t lineNum = 0; UChar bufferU[1024]; uint32_t first = 0; while (fgets(buffer, 1024, testFile) != NULL) { if(*buffer == 0 || buffer[0] == '#') { // Store empty and comment lines so that errors are reported // for the real test file lines. lines[lineNum].buflen = 0; lines[lineNum].buff[0] = 0; } else { int32_t buflen = u_parseString(buffer, bufferU, 1024, &first, &status); lines[lineNum].buflen = buflen; u_memcpy(lines[lineNum].buff, bufferU, buflen); lines[lineNum].buff[buflen] = 0; } lineNum++; } fclose(testFile); if(U_FAILURE(status)) { dataerrln("Couldn't read the test file!"); return; } UVersionInfo uniVersion; static const UVersionInfo v62 = { 6, 2, 0, 0 }; u_getUnicodeVersion(uniVersion); UBool isAtLeastUCA62 = uprv_memcmp(uniVersion, v62, 4) >= 0; LocalPointer coll(Collator::createInstance(Locale::getRoot(), status)); if(U_FAILURE(status)) { errcheckln(status, "Couldn't open UCA collator"); return; } coll->setAttribute(UCOL_NORMALIZATION_MODE, UCOL_ON, status); coll->setAttribute(UCOL_CASE_FIRST, UCOL_OFF, status); coll->setAttribute(UCOL_CASE_LEVEL, UCOL_OFF, status); coll->setAttribute(UCOL_STRENGTH, isAtLeastUCA62 ? UCOL_IDENTICAL : UCOL_TERTIARY, status); coll->setAttribute(UCOL_ALTERNATE_HANDLING, UCOL_NON_IGNORABLE, status); int32_t spawnResult = 0; LocalArray tests(new CollatorThreadTest[kCollatorThreadThreads]); logln(UnicodeString("Spawning: ") + kCollatorThreadThreads + " threads * " + kFormatThreadIterations + " iterations each."); int32_t j = 0; for(j = 0; j < kCollatorThreadThreads; j++) { //logln("Setting collator %i", j); tests[j].setCollator(coll.getAlias(), lines.getAlias(), lineNum, isAtLeastUCA62); } for(j = 0; j < kCollatorThreadThreads; j++) { log("%i ", j); spawnResult = tests[j].start(); if(spawnResult != 0) { errln("%s:%d THREAD INFO: thread %d failed to start with status %d", __FILE__, __LINE__, j, spawnResult); return; } } logln("Spawned all"); for(int32_t i=0;ierrln("%s:%d Original string is corrupt.", __FILE__, __LINE__); break; } UnicodeString s1 = *gSharedString; s1 += "cat this"; UnicodeString s2(s1); UnicodeString s3 = *gSharedString; s2 = s3; s3.truncate(12); s2.truncate(0); } fTraceInfo = 2; } }; const UnicodeString *StringThreadTest2::gSharedString = NULL; // ** The actual test function. void MultithreadTest::TestString() { int j; StringThreadTest2::gSharedString = new UnicodeString("This is the original test string."); StringThreadTest2 tests[kStringThreadThreads]; logln(UnicodeString("Spawning: ") + kStringThreadThreads + " threads * " + kStringThreadIterations + " iterations each."); for(j = 0; j < kStringThreadThreads; j++) { int32_t threadStatus = tests[j].start(); if (threadStatus != 0) { errln("%s:%d System Error %d starting thread number %d.", __FILE__, __LINE__, threadStatus, j); } } // Force a failure, to verify test is functioning and can report errors. // const_cast(StringThreadTest2::gSharedString)->setCharAt(5, 'x'); for(j=0; jtransliterate(greekString); if (greekString[0] != 0x64) // 'd'. The whole transliterated string is "diaphoretikous" (accented u). { IntlTest::gTest->errln("%s:%d Transliteration failed.", __FILE__, __LINE__); } } #endif void MultithreadTest::TestAnyTranslit() { #if !UCONFIG_NO_TRANSLITERATION UErrorCode status = U_ZERO_ERROR; LocalPointer tx(Transliterator::createInstance("Any-Latin", UTRANS_FORWARD, status)); if (U_FAILURE(status)) { dataerrln("File %s, Line %d: Error, status = %s", __FILE__, __LINE__, u_errorName(status)); return; } gSharedTranslit = tx.getAlias(); TxThread threads[4]; int32_t i; for (i=0; ierrln("File %s, Line %d: Error, gStartedThreads = %d, gFinishedThreads = %d", __FILE__, __LINE__, gStartedThreads, gFinishedThreads); } umtx_condWait(&gCTConditionVar, &gCTMutex); } gFinishedThreads += gConditionTestOne; fFinished = true; umtx_condBroadcast(&gCTConditionVar); while (gFinishedThreads < NUMTHREADS) { umtx_condWait(&gCTConditionVar, &gCTMutex); } umtx_unlock(&gCTMutex); } void MultithreadTest::TestConditionVariables() { gStartedThreads = 0; gFinishedThreads = 0; int i; umtx_lock(&gCTMutex); CondThread *threads[NUMTHREADS]; for (i=0; istart(); } while (gStartedThreads < NUMTHREADS) { umtx_condWait(&gCTConditionVar, &gCTMutex); } while (gFinishedThreads < NUMTHREADS) { umtx_condWait(&gCTConditionVar, &gCTMutex); } umtx_unlock(&gCTMutex); for (i=0; ifFinished) { errln("File %s, Line %d: Error, threads[%d]->fFinished == false", __FILE__, __LINE__, i); } } for (i=0; ijoin(); delete threads[i]; } } // // Unified Cache Test // // Each thread fetches a pair of objects. There are 8 distinct pairs: // ("en_US", "bs"), ("en_GB", "ca"), ("fr_FR", "ca_AD") etc. // These pairs represent 8 distinct languages // Note that only one value per language gets created in the cache. // In particular each cached value can have multiple keys. static const char *gCacheLocales[] = { "en_US", "en_GB", "fr_FR", "fr", "de", "sr_ME", "sr_BA", "sr_CS"}; static const char *gCacheLocales2[] = { "bs", "ca", "ca_AD", "ca_ES", "en_US", "fi", "ff_CM", "ff_GN"}; static int32_t gObjectsCreated = 0; // protected by gCTMutex static const int32_t CACHE_LOAD = 3; class UCTMultiThreadItem : public SharedObject { public: char *value; UCTMultiThreadItem(const char *x) : value(NULL) { value = uprv_strdup(x); } virtual ~UCTMultiThreadItem() { uprv_free(value); } }; U_NAMESPACE_BEGIN template<> U_EXPORT const UCTMultiThreadItem *LocaleCacheKey::createObject( const void *context, UErrorCode &status) const { const UnifiedCache *cacheContext = (const UnifiedCache *) context; if (uprv_strcmp(fLoc.getLanguage(), fLoc.getName()) != 0) { const UCTMultiThreadItem *result = NULL; if (cacheContext == NULL) { UnifiedCache::getByLocale(fLoc.getLanguage(), result, status); return result; } cacheContext->get(LocaleCacheKey(fLoc.getLanguage()), result, status); return result; } umtx_lock(&gCTMutex); bool firstObject = (gObjectsCreated == 0); if (firstObject) { // Force the first object creation that comes through to wait // until other have completed. Verifies that cache doesn't // deadlock when a creation is slow. // Note that gObjectsCreated needs to be incremeneted from 0 to 1 // early, to keep subsequent threads from entering this path. gObjectsCreated = 1; while (gObjectsCreated < 3) { umtx_condWait(&gCTConditionVar, &gCTMutex); } } umtx_unlock(&gCTMutex); const UCTMultiThreadItem *result = new UCTMultiThreadItem(fLoc.getLanguage()); if (result == NULL) { status = U_MEMORY_ALLOCATION_ERROR; } else { result->addRef(); } // Log that we created an object. The first object was already counted, // don't do it again. umtx_lock(&gCTMutex); if (!firstObject) { gObjectsCreated += 1; } umtx_condBroadcast(&gCTConditionVar); umtx_unlock(&gCTMutex); return result; } U_NAMESPACE_END class UnifiedCacheThread: public SimpleThread { public: UnifiedCacheThread( const UnifiedCache *cache, const char *loc, const char *loc2) : fCache(cache), fLoc(loc), fLoc2(loc2) {}; ~UnifiedCacheThread() {}; void run(); void exerciseByLocale(const Locale &); const UnifiedCache *fCache; Locale fLoc; Locale fLoc2; }; void UnifiedCacheThread::exerciseByLocale(const Locale &locale) { UErrorCode status = U_ZERO_ERROR; const UCTMultiThreadItem *origItem = NULL; fCache->get( LocaleCacheKey(locale), fCache, origItem, status); U_ASSERT(U_SUCCESS(status)); if (uprv_strcmp(locale.getLanguage(), origItem->value)) { IntlTest::gTest->errln( "%s:%d Expected %s, got %s", __FILE__, __LINE__, locale.getLanguage(), origItem->value); } // Fetch the same item again many times. We should always get the same // pointer since this client is already holding onto it for (int32_t i = 0; i < 1000; ++i) { const UCTMultiThreadItem *item = NULL; fCache->get( LocaleCacheKey(locale), fCache, item, status); if (item != origItem) { IntlTest::gTest->errln( "%s:%d Expected to get the same pointer", __FILE__, __LINE__); } if (item != NULL) { item->removeRef(); } } origItem->removeRef(); } void UnifiedCacheThread::run() { // Run the exercise with 2 different locales so that we can exercise // eviction more. If each thread exercises just one locale, then // eviction can't start until the threads end. exerciseByLocale(fLoc); exerciseByLocale(fLoc2); } void MultithreadTest::TestUnifiedCache() { // Start with our own local cache so that we have complete control // and set the eviction policy to evict starting with 2 unused // values UErrorCode status = U_ZERO_ERROR; UnifiedCache::getInstance(status); UnifiedCache cache(status); cache.setEvictionPolicy(2, 0, status); U_ASSERT(U_SUCCESS(status)); gFinishedThreads = 0; gObjectsCreated = 0; UnifiedCacheThread *threads[CACHE_LOAD][UPRV_LENGTHOF(gCacheLocales)]; for (int32_t i=0; istart(); } } for (int32_t i=0; ijoin(); } } // Because of cache eviction, we can't assert exactly how many // distinct objects get created over the course of this run. // However we know that at least 8 objects get created because that // is how many distinct languages we have in our test. if (gObjectsCreated < 8) { errln("%s:%d Too few objects created.", __FILE__, __LINE__); } // We know that each thread cannot create more than 2 objects in // the cache, and there are UPRV_LENGTHOF(gCacheLocales) pairs of // objects fetched from the cache. If the threads run in series because // of eviction, at worst case each thread creates two objects. if (gObjectsCreated > 2 * CACHE_LOAD * UPRV_LENGTHOF(gCacheLocales)) { errln("%s:%d Too many objects created, got %d, expected %d", __FILE__, __LINE__, gObjectsCreated, 2 * CACHE_LOAD * UPRV_LENGTHOF(gCacheLocales)); } assertEquals("unused values", 2, cache.unusedCount()); // clean up threads for (int32_t i=0; itransliterate(s); if (*gTranslitExpected != s) { IntlTest::gTest->errln("%s:%d Transliteration threading failure.", __FILE__, __LINE__); break; } } } void MultithreadTest::TestBreakTranslit() { UErrorCode status = U_ZERO_ERROR; UnicodeString input( "\\u0E42\\u0E14\\u0E22\\u0E1E\\u0E37\\u0E49\\u0E19\\u0E10\\u0E32\\u0E19\\u0E41\\u0E25\\u0E49\\u0E27,"); input = input.unescape(); gTranslitInput = &input; gSharedTransliterator = Transliterator::createInstance( UNICODE_STRING_SIMPLE("Any-Latin; Lower; NFD; [:Diacritic:]Remove; NFC; Latin-ASCII;"), UTRANS_FORWARD, status); if (!gSharedTransliterator) { return; } TSMTHREAD_ASSERT_SUCCESS(status); UnicodeString expected(*gTranslitInput); gSharedTransliterator->transliterate(expected); gTranslitExpected = &expected; BreakTranslitThread threads[4]; for (int i=0; idefaultCenturyStartYear(); } void MultithreadTest::Test20104() { UErrorCode status = U_ZERO_ERROR; Locale loc("hi_IN"); gSharedCalendar = new IndianCalendar(loc, status); assertSuccess("Test20104", status); static constexpr int NUM_THREADS = 4; Test20104Thread threads[NUM_THREADS]; for (auto &thread:threads) { thread.start(); } for (auto &thread:threads) { thread.join(); } delete gSharedCalendar; // Note: failure is reported by Thread Sanitizer. Test itself succeeds. } #endif /* !UCONFIG_NO_FORMATTING */ #endif /* !UCONFIG_NO_TRANSLITERATION */