Searched refs:line_tokens (Results 1 – 1 of 1) sorted by relevance
| /external/libtextclassifier/native/annotator/ |
| D | annotator.cc | 1981 std::vector<Token> line_tokens; in ModelAnnotate() local
1982 line_tokens = selection_feature_processor_->Tokenize(line_str); in ModelAnnotate()
1987 &line_tokens, in ModelAnnotate()
1990 0, static_cast<TokenIndex>(line_tokens.size())}; in ModelAnnotate()
1994 line_tokens, full_line_span)) { in ModelAnnotate()
2000 line_tokens, full_line_span, in ModelAnnotate()
2012 if (!ModelChunk(line_tokens.size(), /*span_of_interest=*/full_line_span, in ModelAnnotate()
2032 TokenSpanToCodepointSpan(line_tokens, chunk); in ModelAnnotate()
2062 line_unicode, line_tokens, detected_text_language_tags, in ModelAnnotate()
2073 if (!ModelClassifyText(line_str, line_tokens, in ModelAnnotate()
[all …]
|