Searched refs:CreateTokenizer (Results 1 – 5 of 5) sorted by relevance
45 std::unique_ptr<Tokenizer> CreateTokenizer( in CreateTokenizer() function71 tokenizer_(CreateTokenizer(options->tokenizer_options(), unilib)), in ActionsFeatureProcessor()
33 std::unique_ptr<Tokenizer> CreateTokenizer(
34 tokenizer_(CreateTokenizer(grammar_rules->tokenizer_options(), unilib)), in GrammarActions()
84 owned_tokenizer_ = CreateTokenizer(model->tokenizer_options(), unilib); in NGramSensitiveModel()
165 std::unique_ptr<BertTokenizer> CreateTokenizer(const PodNerModel *model) { in CreateTokenizer() function187 std::unique_ptr<BertTokenizer> tokenizer = CreateTokenizer(model); in Create()