Searched refs:InternalTokenize (Results 1 – 2 of 2) sorted by relevance
105 return InternalTokenize(text_unicode); in Tokenize()127 return InternalTokenize(text_unicode); in Tokenize()136 std::vector<Token> Tokenizer::InternalTokenize( in InternalTokenize() function in libtextclassifier3::Tokenizer191 std::vector<Token> tokens = InternalTokenize(text); in TokenizeSubstring()
115 std::vector<Token> InternalTokenize(const UnicodeText& text_unicode) const;