Lines Matching refs:UnicodeText
67 const UnicodeText token_word = in SplitTokensOnSelectionBoundaries()
72 std::vector<UnicodeText::const_iterator> split_points; in SplitTokensOnSelectionBoundaries()
116 void FindSubstrings(const UnicodeText& t, const std::set<char32>& codepoints, in FindSubstrings()
118 UnicodeText::const_iterator start = t.begin(); in FindSubstrings()
119 UnicodeText::const_iterator curr = start; in FindSubstrings()
120 UnicodeText::const_iterator end = t.end(); in FindSubstrings()
137 const UnicodeText context_unicode = UTF8ToUnicodeText(context, in StripTokensFromOtherLines()
464 const UnicodeText value = in SupportedCodepointsRatio()
706 const UnicodeText unicode_text = in InternalRetokenize()
712 const UnicodeText unicode_token_value = in InternalRetokenize()
739 void FeatureProcessor::TokenizeSubstring(const UnicodeText& unicode_text, in TokenizeSubstring()
748 UnicodeText::const_iterator it_begin = unicode_text.begin(); in TokenizeSubstring()
752 UnicodeText::const_iterator it_end = unicode_text.begin(); in TokenizeSubstring()