Searched refs:token_text (Results 1 – 6 of 6) sorted by relevance
| /external/libtextclassifier/native/utils/ |
| D | tokenizer-utils.h | 79 const std::string token_text = UnicodeText::UTF8Substring(start_it, it); in TokenizeWithFilter() local
80 if (!token_text.empty()) { in TokenizeWithFilter()
82 Token{token_text, token_start_codepoint, codepoint_idx}); in TokenizeWithFilter()
96 const std::string token_text = in TokenizeWithFilter() local
98 tokens.push_back(Token{token_text, token_start_codepoint, codepoint_idx}); in TokenizeWithFilter()
|
| /external/antlr/runtime/Perl5/lib/ANTLR/Runtime/ |
| D | Parser.pm | 36 my $token_text;
38 $token_text = '<missing EOF>';
41 $token_text = '<missing ' . $self->get_token_names()->[$expected_token_type] . '>';
46 text => $token_text
|
| /external/libtextclassifier/native/annotator/number/ |
| D | number.cc | 197 bool NumberAnnotator::TryParseNumber(const UnicodeText& token_text, in TryParseNumber() argument
201 if (token_text.ToUTF8String().size() >= max_number_of_digits_) { in TryParseNumber()
204 const bool is_double = unilib_->ParseDouble(token_text, parsed_double_value); in TryParseNumber()
233 const UnicodeText token_text = in FindAll() local
241 if (!TryParseNumber(token_text, is_negative, &parsed_int_value, in FindAll()
|
| D | number.h | 102 bool TryParseNumber(const UnicodeText& token_text, bool is_negative,
|
| /external/antlr/runtime/Ruby/test/functional/lexer/ |
| D | properties.rb | 100 lexer.token_text.should == ' '
|
| /external/libtextclassifier/native/annotator/pod_ner/ |
| D | pod-ner-impl.cc | 359 const std::string token_text = in PrepareText() local
367 tokenizer_->TokenizeSingleToken(token_text); in PrepareText()
|