Lines Matching defs:tokenizer
32 void AssertTokenizerResults(std::unique_ptr<BertTokenizer> tokenizer) { in AssertTokenizerResults()
41 auto tokenizer = in TEST() local
48 auto tokenizer = in TEST() local
60 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
66 auto tokenizer = in TEST() local
76 auto tokenizer = in TEST() local
89 auto tokenizer = in TEST() local
110 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
124 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
145 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
166 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
188 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local