Searched refs:token_begin_it (Results 1 – 3 of 3) sorted by relevance
244 auto token_begin_it = context_unicode.begin(); in ICUTokenize() local252 auto token_end_it = token_begin_it; in ICUTokenize()258 for (auto char_it = token_begin_it; char_it < token_end_it; ++char_it) { in ICUTokenize()266 context_unicode.UTF8Substring(token_begin_it, token_end_it); in ICUTokenize()274 token_begin_it = token_end_it; in ICUTokenize()
65 bool NumberAnnotator::IsCJTterm(UnicodeText::const_iterator token_begin_it, in IsCJTterm() argument67 auto token_end_it = token_begin_it; in IsCJTterm()69 for (auto char_it = token_begin_it; char_it < token_end_it; ++char_it) { in IsCJTterm()
107 bool IsCJTterm(UnicodeText::const_iterator token_begin_it,