1 /*
2 * Copyright 2020 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #include "include/core/SkString.h"
9 #include "include/core/SkTypes.h"
10 #include "include/private/SkBitmaskEnum.h"
11 #include "include/private/SkMutex.h"
12 #include "include/private/SkOnce.h"
13 #include "include/private/SkTArray.h"
14 #include "include/private/SkTemplates.h"
15 #include "include/private/SkTo.h"
16 #include "modules/skunicode/include/SkUnicode.h"
17 #include "modules/skunicode/src/SkUnicode_icu.h"
18 #include "modules/skunicode/src/SkUnicode_icu_bidi.h"
19 #include "src/utils/SkUTF.h"
20 #include "include/private/SkTHash.h"
21 #include <unicode/umachine.h>
22 #include <functional>
23 #include <string>
24 #include <utility>
25 #include <vector>
26
27 #if defined(SK_USING_THIRD_PARTY_ICU)
28 #include "SkLoadICU.h"
29 #endif
30
ICULib()31 static const SkICULib* ICULib() {
32 static const auto gICU = SkLoadICULib();
33
34 return gICU.get();
35 }
36
37 // sk_* wrappers for ICU funcs
38 #define SKICU_FUNC(funcname) \
39 template <typename... Args> \
40 auto sk_##funcname(Args&&... args) -> decltype(funcname(std::forward<Args>(args)...)) { \
41 return ICULib()->f_##funcname(std::forward<Args>(args)...); \
42 } \
43
44 SKICU_EMIT_FUNCS
45 #undef SKICU_FUNC
46
errorName(UErrorCode status)47 const char* SkUnicode_IcuBidi::errorName(UErrorCode status) {
48 return sk_u_errorName(status);
49 }
50
bidi_close(UBiDi * bidi)51 void SkUnicode_IcuBidi::bidi_close(UBiDi* bidi) {
52 sk_ubidi_close(bidi);
53 }
bidi_getDirection(const UBiDi * bidi)54 UBiDiDirection SkUnicode_IcuBidi::bidi_getDirection(const UBiDi* bidi) {
55 return sk_ubidi_getDirection(bidi);
56 }
bidi_getLength(const UBiDi * bidi)57 SkBidiIterator::Position SkUnicode_IcuBidi::bidi_getLength(const UBiDi* bidi) {
58 return sk_ubidi_getLength(bidi);
59 }
bidi_getLevelAt(const UBiDi * bidi,int pos)60 SkBidiIterator::Level SkUnicode_IcuBidi::bidi_getLevelAt(const UBiDi* bidi, int pos) {
61 return sk_ubidi_getLevelAt(bidi, pos);
62 }
bidi_openSized(int32_t maxLength,int32_t maxRunCount,UErrorCode * pErrorCode)63 UBiDi* SkUnicode_IcuBidi::bidi_openSized(int32_t maxLength, int32_t maxRunCount, UErrorCode* pErrorCode) {
64 return sk_ubidi_openSized(maxLength, maxRunCount, pErrorCode);
65 }
bidi_setPara(UBiDi * bidi,const UChar * text,int32_t length,UBiDiLevel paraLevel,UBiDiLevel * embeddingLevels,UErrorCode * status)66 void SkUnicode_IcuBidi::bidi_setPara(UBiDi* bidi,
67 const UChar* text,
68 int32_t length,
69 UBiDiLevel paraLevel,
70 UBiDiLevel* embeddingLevels,
71 UErrorCode* status) {
72 return sk_ubidi_setPara(bidi, text, length, paraLevel, embeddingLevels, status);
73 }
bidi_reorderVisual(const SkUnicode::BidiLevel runLevels[],int levelsCount,int32_t logicalFromVisual[])74 void SkUnicode_IcuBidi::bidi_reorderVisual(const SkUnicode::BidiLevel runLevels[],
75 int levelsCount,
76 int32_t logicalFromVisual[]) {
77 sk_ubidi_reorderVisual(runLevels, levelsCount, logicalFromVisual);
78 }
79
sk_ubrk_clone(const UBreakIterator * bi,UErrorCode * status)80 static inline UBreakIterator* sk_ubrk_clone(const UBreakIterator* bi, UErrorCode* status) {
81 const auto* icu = ICULib();
82 SkASSERT(icu->f_ubrk_clone_ || icu->f_ubrk_safeClone_);
83 return icu->f_ubrk_clone_
84 ? icu->f_ubrk_clone_(bi, status)
85 : icu->f_ubrk_safeClone_(bi, nullptr, nullptr, status);
86 }
87
utext_close_wrapper(UText * ut)88 static UText* utext_close_wrapper(UText* ut) {
89 return sk_utext_close(ut);
90 }
ubrk_close_wrapper(UBreakIterator * bi)91 static void ubrk_close_wrapper(UBreakIterator* bi) {
92 sk_ubrk_close(bi);
93 }
94
95 using ICUUText = std::unique_ptr<UText, SkFunctionWrapper<decltype(utext_close),
96 utext_close_wrapper>>;
97 using ICUBreakIterator = std::unique_ptr<UBreakIterator, SkFunctionWrapper<decltype(ubrk_close),
98 ubrk_close_wrapper>>;
99 /** Replaces invalid utf-8 sequences with REPLACEMENT CHARACTER U+FFFD. */
utf8_next(const char ** ptr,const char * end)100 static inline SkUnichar utf8_next(const char** ptr, const char* end) {
101 SkUnichar val = SkUTF::NextUTF8(ptr, end);
102 return val < 0 ? 0xFFFD : val;
103 }
104
convertType(SkUnicode::BreakType type)105 static UBreakIteratorType convertType(SkUnicode::BreakType type) {
106 switch (type) {
107 case SkUnicode::BreakType::kLines: return UBRK_LINE;
108 case SkUnicode::BreakType::kGraphemes: return UBRK_CHARACTER;
109 case SkUnicode::BreakType::kWords: return UBRK_WORD;
110 default:
111 return UBRK_CHARACTER;
112 }
113 }
114
115 class SkBreakIterator_icu : public SkBreakIterator {
116 ICUBreakIterator fBreakIterator;
117 Position fLastResult;
118 public:
SkBreakIterator_icu(ICUBreakIterator iter)119 explicit SkBreakIterator_icu(ICUBreakIterator iter)
120 : fBreakIterator(std::move(iter))
121 , fLastResult(0) {}
first()122 Position first() override { return fLastResult = sk_ubrk_first(fBreakIterator.get()); }
current()123 Position current() override { return fLastResult = sk_ubrk_current(fBreakIterator.get()); }
next()124 Position next() override { return fLastResult = sk_ubrk_next(fBreakIterator.get()); }
status()125 Status status() override { return sk_ubrk_getRuleStatus(fBreakIterator.get()); }
isDone()126 bool isDone() override { return fLastResult == UBRK_DONE; }
127
setText(const char utftext8[],int utf8Units)128 bool setText(const char utftext8[], int utf8Units) override {
129 UErrorCode status = U_ZERO_ERROR;
130 ICUUText text(sk_utext_openUTF8(nullptr, &utftext8[0], utf8Units, &status));
131
132 if (U_FAILURE(status)) {
133 SkDEBUGF("Break error: %s", sk_u_errorName(status));
134 return false;
135 }
136 SkASSERT(text);
137 sk_ubrk_setUText(fBreakIterator.get(), text.get(), &status);
138 if (U_FAILURE(status)) {
139 SkDEBUGF("Break error: %s", sk_u_errorName(status));
140 return false;
141 }
142 fLastResult = 0;
143 return true;
144 }
setText(const char16_t utftext16[],int utf16Units)145 bool setText(const char16_t utftext16[], int utf16Units) override {
146 UErrorCode status = U_ZERO_ERROR;
147 ICUUText text(sk_utext_openUChars(nullptr, reinterpret_cast<const UChar*>(&utftext16[0]),
148 utf16Units, &status));
149
150 if (U_FAILURE(status)) {
151 SkDEBUGF("Break error: %s", sk_u_errorName(status));
152 return false;
153 }
154 SkASSERT(text);
155 sk_ubrk_setUText(fBreakIterator.get(), text.get(), &status);
156 if (U_FAILURE(status)) {
157 SkDEBUGF("Break error: %s", sk_u_errorName(status));
158 return false;
159 }
160 fLastResult = 0;
161 return true;
162 }
163 };
164
165 class SkIcuBreakIteratorCache {
166 SkTHashMap<SkUnicode::BreakType, ICUBreakIterator> fBreakCache;
167 SkMutex fBreakCacheMutex;
168
169 public:
get()170 static SkIcuBreakIteratorCache& get() {
171 static SkIcuBreakIteratorCache instance;
172 return instance;
173 }
174
makeBreakIterator(SkUnicode::BreakType type)175 ICUBreakIterator makeBreakIterator(SkUnicode::BreakType type) {
176 UErrorCode status = U_ZERO_ERROR;
177 ICUBreakIterator* cachedIterator;
178 {
179 SkAutoMutexExclusive lock(fBreakCacheMutex);
180 cachedIterator = fBreakCache.find(type);
181 if (!cachedIterator) {
182 ICUBreakIterator newIterator(sk_ubrk_open(convertType(type), sk_uloc_getDefault(),
183 nullptr, 0, &status));
184 if (U_FAILURE(status)) {
185 SkDEBUGF("Break error: %s", sk_u_errorName(status));
186 } else {
187 cachedIterator = fBreakCache.set(type, std::move(newIterator));
188 }
189 }
190 }
191 ICUBreakIterator iterator;
192 if (cachedIterator) {
193 iterator.reset(sk_ubrk_clone(cachedIterator->get(), &status));
194 if (U_FAILURE(status)) {
195 SkDEBUGF("Break error: %s", sk_u_errorName(status));
196 }
197 }
198 return iterator;
199 }
200 };
201
202 class SkUnicode_icu : public SkUnicode {
203
copy()204 std::unique_ptr<SkUnicode> copy() override {
205 return std::make_unique<SkUnicode_icu>();
206 }
207
extractWords(uint16_t utf16[],int utf16Units,const char * locale,std::vector<Position> * words)208 static bool extractWords(uint16_t utf16[], int utf16Units, const char* locale, std::vector<Position>* words) {
209
210 UErrorCode status = U_ZERO_ERROR;
211
212 ICUBreakIterator iterator = SkIcuBreakIteratorCache::get().makeBreakIterator(BreakType::kWords);
213 if (!iterator) {
214 SkDEBUGF("Break error: %s", sk_u_errorName(status));
215 return false;
216 }
217 SkASSERT(iterator);
218
219 ICUUText utf16UText(sk_utext_openUChars(nullptr, (UChar*)utf16, utf16Units, &status));
220 if (U_FAILURE(status)) {
221 SkDEBUGF("Break error: %s", sk_u_errorName(status));
222 return false;
223 }
224
225 sk_ubrk_setUText(iterator.get(), utf16UText.get(), &status);
226 if (U_FAILURE(status)) {
227 SkDEBUGF("Break error: %s", sk_u_errorName(status));
228 return false;
229 }
230
231 // Get the words
232 int32_t pos = sk_ubrk_first(iterator.get());
233 while (pos != UBRK_DONE) {
234 words->emplace_back(pos);
235 pos = sk_ubrk_next(iterator.get());
236 }
237
238 return true;
239 }
240
extractPositions(const char utf8[],int utf8Units,BreakType type,std::function<void (int,int)> setBreak)241 static bool extractPositions
242 (const char utf8[], int utf8Units, BreakType type, std::function<void(int, int)> setBreak) {
243
244 UErrorCode status = U_ZERO_ERROR;
245 ICUUText text(sk_utext_openUTF8(nullptr, &utf8[0], utf8Units, &status));
246
247 if (U_FAILURE(status)) {
248 SkDEBUGF("Break error: %s", sk_u_errorName(status));
249 return false;
250 }
251 SkASSERT(text);
252
253 ICUBreakIterator iterator = SkIcuBreakIteratorCache::get().makeBreakIterator(type);
254 if (!iterator) {
255 return false;
256 }
257
258 sk_ubrk_setUText(iterator.get(), text.get(), &status);
259 if (U_FAILURE(status)) {
260 SkDEBUGF("Break error: %s", sk_u_errorName(status));
261 return false;
262 }
263
264 auto iter = iterator.get();
265 int32_t pos = sk_ubrk_first(iter);
266 while (pos != UBRK_DONE) {
267 int s = type == SkUnicode::BreakType::kLines
268 ? UBRK_LINE_SOFT
269 : sk_ubrk_getRuleStatus(iter);
270 setBreak(pos, s);
271 pos = sk_ubrk_next(iter);
272 }
273
274 if (type == SkUnicode::BreakType::kLines) {
275 // This is a workaround for https://bugs.chromium.org/p/skia/issues/detail?id=10715
276 // (ICU line break iterator does not work correctly on Thai text with new lines)
277 // So, we only use the iterator to collect soft line breaks and
278 // scan the text for all hard line breaks ourselves
279 const char* end = utf8 + utf8Units;
280 const char* ch = utf8;
281 while (ch < end) {
282 auto unichar = utf8_next(&ch, end);
283 if (isHardLineBreak(unichar)) {
284 setBreak(ch - utf8, UBRK_LINE_HARD);
285 }
286 }
287 }
288 return true;
289 }
290
isControl(SkUnichar utf8)291 static bool isControl(SkUnichar utf8) {
292 return sk_u_iscntrl(utf8);
293 }
294
isWhitespace(SkUnichar utf8)295 static bool isWhitespace(SkUnichar utf8) {
296 return sk_u_isWhitespace(utf8);
297 }
298
isSpace(SkUnichar utf8)299 static bool isSpace(SkUnichar utf8) {
300 return sk_u_isspace(utf8);
301 }
302
isTabulation(SkUnichar utf8)303 static bool isTabulation(SkUnichar utf8) {
304 return utf8 == '\t';
305 }
306
isHardBreak(SkUnichar utf8)307 static bool isHardBreak(SkUnichar utf8) {
308 auto property = sk_u_getIntPropertyValue(utf8, UCHAR_LINE_BREAK);
309 return property == U_LB_LINE_FEED || property == U_LB_MANDATORY_BREAK;
310 }
311
isIdeographic(SkUnichar unichar)312 static bool isIdeographic(SkUnichar unichar) {
313 return sk_u_hasBinaryProperty(unichar, UCHAR_IDEOGRAPHIC);
314 }
315
316 #ifdef OHOS_SUPPORT
isGraphemeExtend(SkUnichar unichar)317 static bool isGraphemeExtend(SkUnichar unichar) {
318 return sk_u_hasBinaryProperty(unichar, UCHAR_GRAPHEME_EXTEND);
319 }
320 #endif
321
322 public:
~SkUnicode_icu()323 ~SkUnicode_icu() override { }
makeBidiIterator(const uint16_t text[],int count,SkBidiIterator::Direction dir)324 std::unique_ptr<SkBidiIterator> makeBidiIterator(const uint16_t text[], int count,
325 SkBidiIterator::Direction dir) override {
326 return SkUnicode::makeBidiIterator(text, count, dir);
327 }
makeBidiIterator(const char text[],int count,SkBidiIterator::Direction dir)328 std::unique_ptr<SkBidiIterator> makeBidiIterator(const char text[],
329 int count,
330 SkBidiIterator::Direction dir) override {
331 return SkUnicode::makeBidiIterator(text, count, dir);
332 }
makeBreakIterator(const char locale[],BreakType breakType)333 std::unique_ptr<SkBreakIterator> makeBreakIterator(const char locale[],
334 BreakType breakType) override {
335 UErrorCode status = U_ZERO_ERROR;
336 ICUBreakIterator iterator(sk_ubrk_open(convertType(breakType), locale, nullptr, 0,
337 &status));
338 if (U_FAILURE(status)) {
339 SkDEBUGF("Break error: %s", sk_u_errorName(status));
340 return nullptr;
341 }
342 return std::unique_ptr<SkBreakIterator>(new SkBreakIterator_icu(std::move(iterator)));
343 }
makeBreakIterator(BreakType breakType)344 std::unique_ptr<SkBreakIterator> makeBreakIterator(BreakType breakType) override {
345 return makeBreakIterator(sk_uloc_getDefault(), breakType);
346 }
347
isHardLineBreak(SkUnichar utf8)348 static bool isHardLineBreak(SkUnichar utf8) {
349 auto property = sk_u_getIntPropertyValue(utf8, UCHAR_LINE_BREAK);
350 return property == U_LB_LINE_FEED || property == U_LB_MANDATORY_BREAK;
351 }
352
toUpper(const SkString & str)353 SkString toUpper(const SkString& str) override {
354 // Convert to UTF16 since that's what ICU wants.
355 auto str16 = SkUnicode::convertUtf8ToUtf16(str.c_str(), str.size());
356
357 UErrorCode icu_err = U_ZERO_ERROR;
358 const auto upper16len = sk_u_strToUpper(nullptr, 0, (UChar*)(str16.c_str()), str16.size(),
359 nullptr, &icu_err);
360 if (icu_err != U_BUFFER_OVERFLOW_ERROR || upper16len <= 0) {
361 return SkString();
362 }
363
364 SkAutoSTArray<128, uint16_t> upper16(upper16len);
365 icu_err = U_ZERO_ERROR;
366 sk_u_strToUpper((UChar*)(upper16.get()), SkToS32(upper16.size()),
367 (UChar*)(str16.c_str()), str16.size(),
368 nullptr, &icu_err);
369 SkASSERT(!U_FAILURE(icu_err));
370
371 // ... and back to utf8 'cause that's what we want.
372 return convertUtf16ToUtf8((char16_t*)upper16.get(), upper16.size());
373 }
374
getBidiRegions(const char utf8[],int utf8Units,TextDirection dir,std::vector<BidiRegion> * results)375 bool getBidiRegions(const char utf8[],
376 int utf8Units,
377 TextDirection dir,
378 std::vector<BidiRegion>* results) override {
379 return SkUnicode::extractBidi(utf8, utf8Units, dir, results);
380 }
381
getWords(const char utf8[],int utf8Units,const char * locale,std::vector<Position> * results)382 bool getWords(const char utf8[], int utf8Units, const char* locale, std::vector<Position>* results) override {
383
384 // Convert to UTF16 since we want the results in utf16
385 auto utf16 = convertUtf8ToUtf16(utf8, utf8Units);
386 return SkUnicode_icu::extractWords((uint16_t*)utf16.c_str(), utf16.size(), locale, results);
387 }
388
computeCodeUnitFlags(char utf8[],int utf8Units,bool replaceTabs,SkTArray<SkUnicode::CodeUnitFlags,true> * results)389 bool computeCodeUnitFlags(char utf8[], int utf8Units, bool replaceTabs,
390 SkTArray<SkUnicode::CodeUnitFlags, true>* results) override {
391 results->reset();
392 results->push_back_n(utf8Units + 1, CodeUnitFlags::kNoCodeUnitFlag);
393
394 SkUnicode_icu::extractPositions(utf8, utf8Units, BreakType::kLines, [&](int pos,
395 int status) {
396 (*results)[pos] |= status == UBRK_LINE_HARD
397 ? CodeUnitFlags::kHardLineBreakBefore
398 : CodeUnitFlags::kSoftLineBreakBefore;
399 });
400
401 SkUnicode_icu::extractPositions(utf8, utf8Units, BreakType::kGraphemes, [&](int pos,
402 int status) {
403 (*results)[pos] |= CodeUnitFlags::kGraphemeStart;
404 });
405
406 const char* current = utf8;
407 const char* end = utf8 + utf8Units;
408 while (current < end) {
409 auto before = current - utf8;
410 SkUnichar unichar = SkUTF::NextUTF8(¤t, end);
411 if (unichar < 0) unichar = 0xFFFD;
412 auto after = current - utf8;
413 if (replaceTabs && SkUnicode_icu::isTabulation(unichar)) {
414 results->at(before) |= SkUnicode::kTabulation;
415 if (replaceTabs) {
416 unichar = ' ';
417 utf8[before] = ' ';
418 }
419 }
420 for (auto i = before; i < after; ++i) {
421 if (SkUnicode_icu::isSpace(unichar)) {
422 results->at(i) |= SkUnicode::kPartOfIntraWordBreak;
423 }
424 if (SkUnicode_icu::isWhitespace(unichar)) {
425 results->at(i) |= SkUnicode::kPartOfWhiteSpaceBreak;
426 }
427 if (SkUnicode_icu::isControl(unichar)) {
428 results->at(i) |= SkUnicode::kControl;
429 }
430 if (SkUnicode_icu::isIdeographic(unichar)) {
431 results->at(i) |= SkUnicode::kIdeographic;
432 }
433 }
434
435 #ifdef OHOS_SUPPORT
436 if (SkUnicode_icu::isGraphemeExtend(unichar)) {
437 // Current unichar is a combining one.
438 results->at(before) |= SkUnicode::kCombine;
439 }
440 #endif
441 }
442
443 return true;
444 }
445
computeCodeUnitFlags(char16_t utf16[],int utf16Units,bool replaceTabs,SkTArray<SkUnicode::CodeUnitFlags,true> * results)446 bool computeCodeUnitFlags(char16_t utf16[], int utf16Units, bool replaceTabs,
447 SkTArray<SkUnicode::CodeUnitFlags, true>* results) override {
448 results->reset();
449 results->push_back_n(utf16Units + 1, CodeUnitFlags::kNoCodeUnitFlag);
450
451 // Get white spaces
452 this->forEachCodepoint((char16_t*)&utf16[0], utf16Units,
453 [results, replaceTabs, &utf16](SkUnichar unichar, int32_t start, int32_t end) {
454 for (auto i = start; i < end; ++i) {
455 if (replaceTabs && SkUnicode_icu::isTabulation(unichar)) {
456 results->at(i) |= SkUnicode::kTabulation;
457 if (replaceTabs) {
458 unichar = ' ';
459 utf16[start] = ' ';
460 }
461 }
462 if (SkUnicode_icu::isSpace(unichar)) {
463 results->at(i) |= SkUnicode::kPartOfIntraWordBreak;
464 }
465 if (SkUnicode_icu::isWhitespace(unichar)) {
466 results->at(i) |= SkUnicode::kPartOfWhiteSpaceBreak;
467 }
468 if (SkUnicode_icu::isControl(unichar)) {
469 results->at(i) |= SkUnicode::kControl;
470 }
471 }
472 });
473 // Get graphemes
474 this->forEachBreak((char16_t*)&utf16[0],
475 utf16Units,
476 SkUnicode::BreakType::kGraphemes,
477 [results](SkBreakIterator::Position pos, SkBreakIterator::Status) {
478 (*results)[pos] |= CodeUnitFlags::kGraphemeStart;
479 });
480 // Get line breaks
481 this->forEachBreak(
482 (char16_t*)&utf16[0],
483 utf16Units,
484 SkUnicode::BreakType::kLines,
485 [results](SkBreakIterator::Position pos, SkBreakIterator::Status status) {
486 if (status ==
487 (SkBreakIterator::Status)SkUnicode::LineBreakType::kHardLineBreak) {
488 // Hard line breaks clears off all the other flags
489 // TODO: Treat \n as a formatting mark and do not pass it to SkShaper
490 (*results)[pos-1] = CodeUnitFlags::kHardLineBreakBefore;
491 } else {
492 (*results)[pos] |= CodeUnitFlags::kSoftLineBreakBefore;
493 }
494 });
495
496 return true;
497 }
498
reorderVisual(const BidiLevel runLevels[],int levelsCount,int32_t logicalFromVisual[])499 void reorderVisual(const BidiLevel runLevels[],
500 int levelsCount,
501 int32_t logicalFromVisual[]) override {
502 SkUnicode_IcuBidi::bidi_reorderVisual(runLevels, levelsCount, logicalFromVisual);
503 }
504 };
505
MakeIcuBasedUnicode()506 std::unique_ptr<SkUnicode> SkUnicode::MakeIcuBasedUnicode() {
507 #if defined(SK_USING_THIRD_PARTY_ICU)
508 if (!SkLoadICU()) {
509 static SkOnce once;
510 once([] { SkDEBUGF("SkLoadICU() failed!\n"); });
511 return nullptr;
512 }
513 #endif
514
515 return ICULib()
516 ? std::make_unique<SkUnicode_icu>()
517 : nullptr;
518 }
519