Searched refs:tokenizer_ (Results 1 – 3 of 3) sorted by relevance
242 tokenizer_(input_stream, &tokenizer_error_collector_), in ParserImpl()252 tokenizer_.set_allow_f_after_float(true); in ParserImpl()255 tokenizer_.set_comment_style(io::Tokenizer::SH_COMMENT_STYLE); in ParserImpl()258 tokenizer_.set_require_space_after_number(false); in ParserImpl()259 tokenizer_.set_allow_multiline_strings(true); in ParserImpl()263 tokenizer_.Next(); in ParserImpl()333 ReportError(tokenizer_.current().line, tokenizer_.current().column, in ReportError()340 ReportWarning(tokenizer_.current().line, tokenizer_.current().column, in ReportWarning()378 int start_line = tokenizer_.current().line; in ConsumeField()379 int start_column = tokenizer_.current().column; in ConsumeField()[all …]
97 tokenizer_({options.tokenization_codepoint_config().begin(), in FeatureProcessor()252 Tokenizer tokenizer_; variable
187 return tokenizer_.Tokenize(utf8_text); in Tokenize()204 return tokenizer_.Tokenize(utf8_text); in Tokenize()760 std::vector<Token> tokens = tokenizer_.Tokenize(text); in TokenizeSubstring()