1 /*
2 ************************************************************************************
3 * Copyright (C) 2006-2012, International Business Machines Corporation
4 * and others. All Rights Reserved.
5 ************************************************************************************
6 */
7
8 #include "unicode/utypes.h"
9
10 #if !UCONFIG_NO_BREAK_ITERATION
11
12 #include "brkeng.h"
13 #include "dictbe.h"
14 #include "unicode/uchar.h"
15 #include "unicode/uniset.h"
16 #include "unicode/chariter.h"
17 #include "unicode/ures.h"
18 #include "unicode/udata.h"
19 #include "unicode/putil.h"
20 #include "unicode/ustring.h"
21 #include "unicode/uscript.h"
22 #include "unicode/ucharstrie.h"
23 #include "unicode/bytestrie.h"
24 #include "charstr.h"
25 #include "dictionarydata.h"
26 #include "uvector.h"
27 #include "umutex.h"
28 #include "uresimp.h"
29 #include "ubrkimpl.h"
30
31 U_NAMESPACE_BEGIN
32
33 /*
34 ******************************************************************
35 */
36
LanguageBreakEngine()37 LanguageBreakEngine::LanguageBreakEngine() {
38 }
39
~LanguageBreakEngine()40 LanguageBreakEngine::~LanguageBreakEngine() {
41 }
42
43 /*
44 ******************************************************************
45 */
46
LanguageBreakFactory()47 LanguageBreakFactory::LanguageBreakFactory() {
48 }
49
~LanguageBreakFactory()50 LanguageBreakFactory::~LanguageBreakFactory() {
51 }
52
53 /*
54 ******************************************************************
55 */
56
UnhandledEngine(UErrorCode &)57 UnhandledEngine::UnhandledEngine(UErrorCode &/*status*/) {
58 for (int32_t i = 0; i < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])); ++i) {
59 fHandled[i] = 0;
60 }
61 }
62
~UnhandledEngine()63 UnhandledEngine::~UnhandledEngine() {
64 for (int32_t i = 0; i < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])); ++i) {
65 if (fHandled[i] != 0) {
66 delete fHandled[i];
67 }
68 }
69 }
70
71 UBool
handles(UChar32 c,int32_t breakType) const72 UnhandledEngine::handles(UChar32 c, int32_t breakType) const {
73 return (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))
74 && fHandled[breakType] != 0 && fHandled[breakType]->contains(c));
75 }
76
77 int32_t
findBreaks(UText * text,int32_t startPos,int32_t endPos,UBool reverse,int32_t breakType,UStack &) const78 UnhandledEngine::findBreaks( UText *text,
79 int32_t startPos,
80 int32_t endPos,
81 UBool reverse,
82 int32_t breakType,
83 UStack &/*foundBreaks*/ ) const {
84 if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) {
85 UChar32 c = utext_current32(text);
86 if (reverse) {
87 while((int32_t)utext_getNativeIndex(text) > startPos && fHandled[breakType]->contains(c)) {
88 c = utext_previous32(text);
89 }
90 }
91 else {
92 while((int32_t)utext_getNativeIndex(text) < endPos && fHandled[breakType]->contains(c)) {
93 utext_next32(text); // TODO: recast loop to work with post-increment operations.
94 c = utext_current32(text);
95 }
96 }
97 }
98 return 0;
99 }
100
101 void
handleCharacter(UChar32 c,int32_t breakType)102 UnhandledEngine::handleCharacter(UChar32 c, int32_t breakType) {
103 if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) {
104 if (fHandled[breakType] == 0) {
105 fHandled[breakType] = new UnicodeSet();
106 if (fHandled[breakType] == 0) {
107 return;
108 }
109 }
110 if (!fHandled[breakType]->contains(c)) {
111 UErrorCode status = U_ZERO_ERROR;
112 // Apply the entire script of the character.
113 int32_t script = u_getIntPropertyValue(c, UCHAR_SCRIPT);
114 fHandled[breakType]->applyIntPropertyValue(UCHAR_SCRIPT, script, status);
115 }
116 }
117 }
118
119 /*
120 ******************************************************************
121 */
122
ICULanguageBreakFactory(UErrorCode &)123 ICULanguageBreakFactory::ICULanguageBreakFactory(UErrorCode &/*status*/) {
124 fEngines = 0;
125 }
126
~ICULanguageBreakFactory()127 ICULanguageBreakFactory::~ICULanguageBreakFactory() {
128 if (fEngines != 0) {
129 delete fEngines;
130 }
131 }
132
133 U_NAMESPACE_END
134 U_CDECL_BEGIN
_deleteEngine(void * obj)135 static void U_CALLCONV _deleteEngine(void *obj) {
136 delete (const icu::LanguageBreakEngine *) obj;
137 }
138 U_CDECL_END
139 U_NAMESPACE_BEGIN
140
141 const LanguageBreakEngine *
getEngineFor(UChar32 c,int32_t breakType)142 ICULanguageBreakFactory::getEngineFor(UChar32 c, int32_t breakType) {
143 UBool needsInit;
144 int32_t i;
145 const LanguageBreakEngine *lbe = NULL;
146 UErrorCode status = U_ZERO_ERROR;
147
148 // TODO: The global mutex should not be used.
149 // The global mutex should only be used for short periods.
150 // A ICULanguageBreakFactory specific mutex should be used.
151 umtx_lock(NULL);
152 needsInit = (UBool)(fEngines == NULL);
153 if (!needsInit) {
154 i = fEngines->size();
155 while (--i >= 0) {
156 lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i));
157 if (lbe != NULL && lbe->handles(c, breakType)) {
158 break;
159 }
160 lbe = NULL;
161 }
162 }
163 umtx_unlock(NULL);
164
165 if (lbe != NULL) {
166 return lbe;
167 }
168
169 if (needsInit) {
170 UStack *engines = new UStack(_deleteEngine, NULL, status);
171 if (U_SUCCESS(status) && engines == NULL) {
172 status = U_MEMORY_ALLOCATION_ERROR;
173 }
174 else if (U_FAILURE(status)) {
175 delete engines;
176 engines = NULL;
177 }
178 else {
179 umtx_lock(NULL);
180 if (fEngines == NULL) {
181 fEngines = engines;
182 engines = NULL;
183 }
184 umtx_unlock(NULL);
185 delete engines;
186 }
187 }
188
189 if (fEngines == NULL) {
190 return NULL;
191 }
192
193 // We didn't find an engine the first time through, or there was no
194 // stack. Create an engine.
195 const LanguageBreakEngine *newlbe = loadEngineFor(c, breakType);
196
197 // Now get the lock, and see if someone else has created it in the
198 // meantime
199 umtx_lock(NULL);
200 i = fEngines->size();
201 while (--i >= 0) {
202 lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i));
203 if (lbe != NULL && lbe->handles(c, breakType)) {
204 break;
205 }
206 lbe = NULL;
207 }
208 if (lbe == NULL && newlbe != NULL) {
209 fEngines->push((void *)newlbe, status);
210 lbe = newlbe;
211 newlbe = NULL;
212 }
213 umtx_unlock(NULL);
214
215 delete newlbe;
216
217 return lbe;
218 }
219
220 const LanguageBreakEngine *
loadEngineFor(UChar32 c,int32_t breakType)221 ICULanguageBreakFactory::loadEngineFor(UChar32 c, int32_t breakType) {
222 UErrorCode status = U_ZERO_ERROR;
223 UScriptCode code = uscript_getScript(c, &status);
224 if (U_SUCCESS(status)) {
225 DictionaryMatcher *m = loadDictionaryMatcherFor(code, breakType);
226 if (m != NULL) {
227 const LanguageBreakEngine *engine = NULL;
228 switch(code) {
229 case USCRIPT_THAI:
230 engine = new ThaiBreakEngine(m, status);
231 break;
232 case USCRIPT_KHMER:
233 engine = new KhmerBreakEngine(m, status);
234 break;
235
236 #if !UCONFIG_NO_NORMALIZATION
237 // CJK not available w/o normalization
238 case USCRIPT_HANGUL:
239 engine = new CjkBreakEngine(m, kKorean, status);
240 break;
241
242 // use same BreakEngine and dictionary for both Chinese and Japanese
243 case USCRIPT_HIRAGANA:
244 case USCRIPT_KATAKANA:
245 case USCRIPT_HAN:
246 engine = new CjkBreakEngine(m, kChineseJapanese, status);
247 break;
248 #if 0
249 // TODO: Have to get some characters with script=common handled
250 // by CjkBreakEngine (e.g. U+309B). Simply subjecting
251 // them to CjkBreakEngine does not work. The engine has to
252 // special-case them.
253 case USCRIPT_COMMON:
254 {
255 UBlockCode block = ublock_getCode(code);
256 if (block == UBLOCK_HIRAGANA || block == UBLOCK_KATAKANA)
257 engine = new CjkBreakEngine(dict, kChineseJapanese, status);
258 break;
259 }
260 #endif
261 #endif
262
263 default:
264 break;
265 }
266 if (engine == NULL) {
267 delete m;
268 }
269 else if (U_FAILURE(status)) {
270 delete engine;
271 engine = NULL;
272 }
273 return engine;
274 }
275 }
276 return NULL;
277 }
278
279 DictionaryMatcher *
loadDictionaryMatcherFor(UScriptCode script,int32_t)280 ICULanguageBreakFactory::loadDictionaryMatcherFor(UScriptCode script, int32_t /* brkType */) {
281 UErrorCode status = U_ZERO_ERROR;
282 // open root from brkitr tree.
283 UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, "", &status);
284 b = ures_getByKeyWithFallback(b, "dictionaries", b, &status);
285 int32_t dictnlength = 0;
286 const UChar *dictfname =
287 ures_getStringByKeyWithFallback(b, uscript_getShortName(script), &dictnlength, &status);
288 if (U_FAILURE(status)) {
289 ures_close(b);
290 return NULL;
291 }
292 CharString dictnbuf;
293 CharString ext;
294 const UChar *extStart = u_memrchr(dictfname, 0x002e, dictnlength); // last dot
295 if (extStart != NULL) {
296 int32_t len = (int32_t)(extStart - dictfname);
297 ext.appendInvariantChars(UnicodeString(FALSE, extStart + 1, dictnlength - len - 1), status);
298 dictnlength = len;
299 }
300 dictnbuf.appendInvariantChars(UnicodeString(FALSE, dictfname, dictnlength), status);
301 ures_close(b);
302
303 UDataMemory *file = udata_open(U_ICUDATA_BRKITR, ext.data(), dictnbuf.data(), &status);
304 if (U_SUCCESS(status)) {
305 // build trie
306 const uint8_t *data = (const uint8_t *)udata_getMemory(file);
307 const int32_t *indexes = (const int32_t *)data;
308 const int32_t offset = indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
309 const int32_t trieType = indexes[DictionaryData::IX_TRIE_TYPE] & DictionaryData::TRIE_TYPE_MASK;
310 DictionaryMatcher *m = NULL;
311 if (trieType == DictionaryData::TRIE_TYPE_BYTES) {
312 const int32_t transform = indexes[DictionaryData::IX_TRANSFORM];
313 const char *characters = (const char *)(data + offset);
314 m = new BytesDictionaryMatcher(characters, transform, file);
315 }
316 else if (trieType == DictionaryData::TRIE_TYPE_UCHARS) {
317 const UChar *characters = (const UChar *)(data + offset);
318 m = new UCharsDictionaryMatcher(characters, file);
319 }
320 if (m == NULL) {
321 // no matcher exists to take ownership - either we are an invalid
322 // type or memory allocation failed
323 udata_close(file);
324 }
325 return m;
326 } else if (dictfname != NULL) {
327 // we don't have a dictionary matcher.
328 // returning NULL here will cause us to fail to find a dictionary break engine, as expected
329 status = U_ZERO_ERROR;
330 return NULL;
331 }
332 return NULL;
333 }
334
335 U_NAMESPACE_END
336
337 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
338