Searched refs:token_end_it (Results 1 – 2 of 2) sorted by relevance
252 auto token_end_it = token_begin_it; in ICUTokenize() local253 std::advance(token_end_it, token_length); in ICUTokenize()254 TC3_CHECK(token_end_it <= context_unicode.end()); in ICUTokenize()258 for (auto char_it = token_begin_it; char_it < token_end_it; ++char_it) { in ICUTokenize()266 context_unicode.UTF8Substring(token_begin_it, token_end_it); in ICUTokenize()274 token_begin_it = token_end_it; in ICUTokenize()
67 auto token_end_it = token_begin_it; in IsCJTterm() local68 std::advance(token_end_it, token_length); in IsCJTterm()69 for (auto char_it = token_begin_it; char_it < token_end_it; ++char_it) { in IsCJTterm()