1 // Copyright 2011 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/i18n/break_iterator.h"
6
7 #include <stdint.h>
8 #include <ostream>
9
10 #include "base/check.h"
11 #include "base/lazy_instance.h"
12 #include "base/memory/raw_ptr.h"
13 #include "base/notreached.h"
14 #include "base/synchronization/lock.h"
15 #include "third_party/icu/source/common/unicode/ubrk.h"
16 #include "third_party/icu/source/common/unicode/uchar.h"
17 #include "third_party/icu/source/common/unicode/ustring.h"
18
19 namespace base {
20 namespace i18n {
21
22 namespace {
23
24 // We found the usage pattern of break iterator is to create, use and destroy.
25 // The following cache support multiple break iterator in the same thread and
26 // also optimize to not create break iterator many time. For each kind of break
27 // iterator (character, word, line and sentence, but NOT rule), we keep one of
28 // them in the main_ and lease it out. If some other code request a lease
29 // before |main_| is returned, we create a new instance of the iterator.
30 // This will keep at most 4 break iterators (one for each kind) unreleased until
31 // the program destruction time.
32 template <UBreakIteratorType break_type>
33 class DefaultLocaleBreakIteratorCache {
34 public:
DefaultLocaleBreakIteratorCache()35 DefaultLocaleBreakIteratorCache() {
36 main_ = UBreakIteratorPtr(
37 ubrk_open(break_type, nullptr, nullptr, 0, &main_status_));
38 if (U_FAILURE(main_status_)) {
39 NOTREACHED() << "ubrk_open failed for type " << break_type
40 << " with error " << main_status_;
41 }
42 }
Lease(UErrorCode & status)43 UBreakIteratorPtr Lease(UErrorCode& status) {
44 if (U_FAILURE(status)) {
45 return nullptr;
46 }
47 if (U_FAILURE(main_status_)) {
48 status = main_status_;
49 return nullptr;
50 }
51 {
52 AutoLock scoped_lock(lock_);
53 if (main_) {
54 return std::move(main_);
55 }
56 }
57
58 // The main_ is already leased out to some other places, return a new
59 // object instead.
60 UBreakIteratorPtr result(
61 ubrk_open(break_type, nullptr, nullptr, 0, &status));
62 if (U_FAILURE(status)) {
63 NOTREACHED() << "ubrk_open failed for type " << break_type
64 << " with error " << status;
65 }
66 return result;
67 }
68
Return(UBreakIteratorPtr item)69 void Return(UBreakIteratorPtr item) {
70 AutoLock scoped_lock(lock_);
71 if (!main_) {
72 main_ = std::move(item);
73 }
74 }
75
76 private:
77 UErrorCode main_status_ = U_ZERO_ERROR;
78 UBreakIteratorPtr main_ GUARDED_BY(lock_);
79 Lock lock_;
80 };
81
82 static LazyInstance<DefaultLocaleBreakIteratorCache<UBRK_CHARACTER>>::Leaky
83 char_break_cache = LAZY_INSTANCE_INITIALIZER;
84 static LazyInstance<DefaultLocaleBreakIteratorCache<UBRK_WORD>>::Leaky
85 word_break_cache = LAZY_INSTANCE_INITIALIZER;
86 static LazyInstance<DefaultLocaleBreakIteratorCache<UBRK_SENTENCE>>::Leaky
87 sentence_break_cache = LAZY_INSTANCE_INITIALIZER;
88 static LazyInstance<DefaultLocaleBreakIteratorCache<UBRK_LINE>>::Leaky
89 line_break_cache = LAZY_INSTANCE_INITIALIZER;
90
91 } // namespace
92
operator ()(UBreakIterator * ptr)93 void UBreakIteratorDeleter::operator()(UBreakIterator* ptr) {
94 if (ptr) {
95 ubrk_close(ptr);
96 }
97 }
98
BreakIterator(StringPiece16 str,BreakType break_type)99 BreakIterator::BreakIterator(StringPiece16 str, BreakType break_type)
100 : string_(str), break_type_(break_type) {}
101
BreakIterator(StringPiece16 str,const std::u16string & rules)102 BreakIterator::BreakIterator(StringPiece16 str, const std::u16string& rules)
103 : string_(str), rules_(rules), break_type_(RULE_BASED) {}
104
~BreakIterator()105 BreakIterator::~BreakIterator() {
106 switch (break_type_) {
107 case RULE_BASED:
108 return;
109 case BREAK_CHARACTER:
110 char_break_cache.Pointer()->Return(std::move(iter_));
111 return;
112 case BREAK_WORD:
113 word_break_cache.Pointer()->Return(std::move(iter_));
114 return;
115 case BREAK_SENTENCE:
116 sentence_break_cache.Pointer()->Return(std::move(iter_));
117 return;
118 case BREAK_LINE:
119 case BREAK_NEWLINE:
120 line_break_cache.Pointer()->Return(std::move(iter_));
121 return;
122 }
123 }
124
Init()125 bool BreakIterator::Init() {
126 UErrorCode status = U_ZERO_ERROR;
127 UParseError parse_error;
128 switch (break_type_) {
129 case BREAK_CHARACTER:
130 iter_ = char_break_cache.Pointer()->Lease(status);
131 break;
132 case BREAK_WORD:
133 iter_ = word_break_cache.Pointer()->Lease(status);
134 break;
135 case BREAK_SENTENCE:
136 iter_ = sentence_break_cache.Pointer()->Lease(status);
137 break;
138 case BREAK_LINE:
139 case BREAK_NEWLINE:
140 iter_ = line_break_cache.Pointer()->Lease(status);
141 break;
142 case RULE_BASED:
143 iter_ = UBreakIteratorPtr(
144 ubrk_openRules(rules_.c_str(), static_cast<int32_t>(rules_.length()),
145 nullptr, 0, &parse_error, &status));
146 if (U_FAILURE(status)) {
147 NOTREACHED() << "ubrk_openRules failed to parse rule string at line "
148 << parse_error.line << ", offset " << parse_error.offset;
149 }
150 break;
151 }
152
153 if (U_FAILURE(status) || iter_ == nullptr) {
154 return false;
155 }
156
157 if (string_.data() != nullptr) {
158 ubrk_setText(iter_.get(), string_.data(),
159 static_cast<int32_t>(string_.size()), &status);
160 if (U_FAILURE(status)) {
161 return false;
162 }
163 }
164
165 // Move the iterator to the beginning of the string.
166 ubrk_first(iter_.get());
167 return true;
168 }
169
Advance()170 bool BreakIterator::Advance() {
171 int32_t pos;
172 int32_t status;
173 prev_ = pos_;
174 switch (break_type_) {
175 case BREAK_CHARACTER:
176 case BREAK_WORD:
177 case BREAK_LINE:
178 case BREAK_SENTENCE:
179 case RULE_BASED:
180 pos = ubrk_next(iter_.get());
181 if (pos == UBRK_DONE) {
182 pos_ = npos;
183 return false;
184 }
185 pos_ = static_cast<size_t>(pos);
186 return true;
187 case BREAK_NEWLINE:
188 do {
189 pos = ubrk_next(iter_.get());
190 if (pos == UBRK_DONE)
191 break;
192 pos_ = static_cast<size_t>(pos);
193 status = ubrk_getRuleStatus(iter_.get());
194 } while (status >= UBRK_LINE_SOFT && status < UBRK_LINE_SOFT_LIMIT);
195 if (pos == UBRK_DONE && prev_ == pos_) {
196 pos_ = npos;
197 return false;
198 }
199 return true;
200 }
201 }
202
SetText(const char16_t * text,const size_t length)203 bool BreakIterator::SetText(const char16_t* text, const size_t length) {
204 UErrorCode status = U_ZERO_ERROR;
205 ubrk_setText(iter_.get(), text, length, &status);
206 pos_ = 0; // implicit when ubrk_setText is done
207 prev_ = npos;
208 if (U_FAILURE(status)) {
209 NOTREACHED() << "ubrk_setText failed";
210 return false;
211 }
212 string_ = StringPiece16(text, length);
213 return true;
214 }
215
IsWord() const216 bool BreakIterator::IsWord() const {
217 return GetWordBreakStatus() == IS_WORD_BREAK;
218 }
219
GetWordBreakStatus() const220 BreakIterator::WordBreakStatus BreakIterator::GetWordBreakStatus() const {
221 int32_t status = ubrk_getRuleStatus(iter_.get());
222 if (break_type_ != BREAK_WORD && break_type_ != RULE_BASED)
223 return IS_LINE_OR_CHAR_BREAK;
224 // In ICU 60, trying to advance past the end of the text does not change
225 // |status| so that |pos_| has to be checked as well as |status|.
226 // See http://bugs.icu-project.org/trac/ticket/13447 .
227 return (status == UBRK_WORD_NONE || pos_ == npos) ? IS_SKIPPABLE_WORD
228 : IS_WORD_BREAK;
229 }
230
IsEndOfWord(size_t position) const231 bool BreakIterator::IsEndOfWord(size_t position) const {
232 if (break_type_ != BREAK_WORD && break_type_ != RULE_BASED)
233 return false;
234
235 UBool boundary = ubrk_isBoundary(iter_.get(), static_cast<int32_t>(position));
236 int32_t status = ubrk_getRuleStatus(iter_.get());
237 return (!!boundary && status != UBRK_WORD_NONE);
238 }
239
IsStartOfWord(size_t position) const240 bool BreakIterator::IsStartOfWord(size_t position) const {
241 if (break_type_ != BREAK_WORD && break_type_ != RULE_BASED)
242 return false;
243
244 UBool boundary = ubrk_isBoundary(iter_.get(), static_cast<int32_t>(position));
245 ubrk_next(iter_.get());
246 int32_t next_status = ubrk_getRuleStatus(iter_.get());
247 return (!!boundary && next_status != UBRK_WORD_NONE);
248 }
249
IsSentenceBoundary(size_t position) const250 bool BreakIterator::IsSentenceBoundary(size_t position) const {
251 if (break_type_ != BREAK_SENTENCE && break_type_ != RULE_BASED)
252 return false;
253
254 return !!ubrk_isBoundary(iter_.get(), static_cast<int32_t>(position));
255 }
256
IsGraphemeBoundary(size_t position) const257 bool BreakIterator::IsGraphemeBoundary(size_t position) const {
258 if (break_type_ != BREAK_CHARACTER)
259 return false;
260
261 return !!ubrk_isBoundary(iter_.get(), static_cast<int32_t>(position));
262 }
263
GetString() const264 std::u16string BreakIterator::GetString() const {
265 return std::u16string(GetStringPiece());
266 }
267
GetStringPiece() const268 StringPiece16 BreakIterator::GetStringPiece() const {
269 DCHECK(prev_ != npos && pos_ != npos);
270 return string_.substr(prev_, pos_ - prev_);
271 }
272
273 } // namespace i18n
274 } // namespace base
275