Searched refs:codepoint_index (Results 1 – 1 of 1) sorted by relevance
| /external/libtextclassifier/native/utils/ |
| D | tokenizer.cc | 140 int codepoint_index = 0; in InternalTokenize() local
144 ++it, ++codepoint_index) { in InternalTokenize()
155 new_token = Token("", codepoint_index, codepoint_index); in InternalTokenize()
165 new_token = Token("", codepoint_index + 1, codepoint_index + 1); in InternalTokenize()
284 int codepoint_index = 0; in NumberTokenize() local
294 int codepoint_index, NumberTokenType token_type, in NumberTokenize()
298 new_token = Token("", codepoint_index, codepoint_index, in NumberTokenize()
308 int codepoint_index, UnicodeText::const_iterator it) { in NumberTokenize() argument
311 result->emplace_back("", codepoint_index, codepoint_index + 1); in NumberTokenize()
314 new_token = Token("", codepoint_index + 1, codepoint_index + 1); in NumberTokenize()
[all …]
|