Searched refs:text_unicode (Results 1 – 6 of 6) sorted by relevance
98 UnicodeText text_unicode = UTF8ToUnicodeText(text, /*do_copy=*/false); in Tokenize() local99 return Tokenize(text_unicode); in Tokenize()102 std::vector<Token> Tokenizer::Tokenize(const UnicodeText& text_unicode) const { in Tokenize()105 return InternalTokenize(text_unicode); in Tokenize()110 if (!ICUTokenize(text_unicode, &result)) { in Tokenize()114 InternalRetokenize(text_unicode, &result); in Tokenize()120 if (!NumberTokenize(text_unicode, &result)) { in Tokenize()127 return InternalTokenize(text_unicode); in Tokenize()137 const UnicodeText& text_unicode) const { in InternalTokenize()143 for (auto it = text_unicode.begin(); it != text_unicode.end(); in InternalTokenize()[all …]
96 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;115 std::vector<Token> InternalTokenize(const UnicodeText& text_unicode) const;127 bool NumberTokenize(const UnicodeText& text_unicode,
65 bool PrepareText(const UnicodeText &text_unicode,
346 bool PodNerAnnotator::PrepareText(const UnicodeText &text_unicode, in PrepareText() argument351 text_unicode.ToUTF8String()); in PrepareText()
127 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;
194 const UnicodeText& text_unicode) const { in Tokenize()195 return tokenizer_.Tokenize(text_unicode); in Tokenize()