Searched refs:tokenizer_ (Results 1 – 9 of 9) sorted by relevance
| /external/protobuf/src/google/protobuf/ |
| D | text_format.cc | 242 tokenizer_(input_stream, &tokenizer_error_collector_), in ParserImpl()
252 tokenizer_.set_allow_f_after_float(true); in ParserImpl()
255 tokenizer_.set_comment_style(io::Tokenizer::SH_COMMENT_STYLE); in ParserImpl()
258 tokenizer_.set_require_space_after_number(false); in ParserImpl()
259 tokenizer_.set_allow_multiline_strings(true); in ParserImpl()
263 tokenizer_.Next(); in ParserImpl()
333 ReportError(tokenizer_.current().line, tokenizer_.current().column, in ReportError()
340 ReportWarning(tokenizer_.current().line, tokenizer_.current().column, in ReportWarning()
378 int start_line = tokenizer_.current().line; in ConsumeField()
379 int start_column = tokenizer_.current().column; in ConsumeField()
[all …]
|
| /external/libtextclassifier/actions/ |
| D | feature-processor.h | 61 const Tokenizer* tokenizer() const { return tokenizer_.get(); } in tokenizer()
65 const std::unique_ptr<Tokenizer> tokenizer_; variable
|
| D | ngram-model.cc | 83 tokenizer_ = owned_tokenizer_.get(); in NGramModel()
85 tokenizer_ = tokenizer; in NGramModel()
150 const std::vector<Token> raw_tokens = tokenizer_->Tokenize(text); in Eval()
|
| D | ngram-model.h | 58 const Tokenizer* tokenizer_; variable
|
| D | feature-processor.cc | 71 tokenizer_(CreateTokenizer(options->tokenizer_options(), unilib)), in ActionsFeatureProcessor()
|
| /external/libtextclassifier/lang_id/ |
| D | lang-id.cc | 174 tokenizer_.Setup(context); in Setup()
209 tokenizer_.Tokenize(text, &sentence); in ComputeScores()
232 TokenizerForLangId tokenizer_; member in libtextclassifier3::mobile::lang_id::LangIdImpl
|
| /external/libtextclassifier/utils/ |
| D | tokenizer_test.cc | 77 tokenizer_ = std::unique_ptr<TestingTokenizer>(new TestingTokenizer( in TestingTokenizerProxy()
84 tokenizer_->FindTokenizationRange(c); in TestFindTokenizationRole()
93 return tokenizer_->Tokenize(utf8_text); in Tokenize()
99 std::unique_ptr<TestingTokenizer> tokenizer_; member in libtextclassifier3::__anonfe576b980111::TestingTokenizerProxy
|
| /external/libtextclassifier/annotator/ |
| D | feature-processor.h | 98 tokenizer_(internal::BuildTokenizer(options, unilib)) { in FeatureProcessor()
285 Tokenizer tokenizer_; variable
|
| D | feature-processor.cc | 192 return tokenizer_.Tokenize(text); in Tokenize()
197 return tokenizer_.Tokenize(text_unicode); in Tokenize()
|