92   std::vector<Token> Tokenize(const std::string& utf8_text) const {  in Tokenize()
174   std::vector<Token> tokens = tokenizer.Tokenize("Hello world!");  in TEST()
177               ElementsAreArray({Token("Hello", 0, 5), Token("world!", 6, 12)}));  in TEST()
209               std::vector<Token>({Token("앨라배마", 0, 4), Token("주", 5, 6),  in TEST()
210                                   Token("전화", 7, 10), Token("(123)", 10, 15),  in TEST()
211                                   Token("456-789", 16, 23),  in TEST()
212                                   Token("웹사이트", 23, 28)}));  in TEST()
343   std::vector<Token> tokens;  in TEST()
353       ElementsAreArray({Token("問", 0, 1),  in TEST()
354                         Token("少", 1, 2),  in TEST()
355                         Token("目", 2, 3),  in TEST()
356                         Token("hello", 4, 9),  in TEST()
357                         Token("木", 10, 11),  in TEST()
358                         Token("輸", 11, 12),  in TEST()
359                         Token("ย", 12, 13),  in TEST()
360                         Token("า", 13, 14),  in TEST()
361                         Token("ม", 14, 15),  in TEST()
362                         Token("き", 15, 16),  in TEST()
363                         Token("ゃ", 16, 17)}));  in TEST()
372   std::vector<Token> tokens = tokenizer.Tokenize("พระบาทสมเด็จพระปรมิ");  in TEST()
375             std::vector<Token>({Token("พระบาท", 0, 6),  in TEST()
376                                 Token("สมเด็จ", 6, 12),  in TEST()
377                                 Token("พระ", 12, 15),  in TEST()
378                                 Token("ปร", 15, 17),  in TEST()
379                                 Token("มิ", 17, 19)}));  in TEST()
387   std::vector<Token> tokens = tokenizer.Tokenize("พระบาท สมเด็จ พระ ปร มิ");  in TEST()
390             std::vector<Token>({Token("พระบาท", 0, 6),  in TEST()
391                                 Token(" ", 6, 7),  in TEST()
392                                 Token("สมเด็จ", 7, 13),  in TEST()
393                                 Token(" ", 13, 14),  in TEST()
394                                 Token("พระ", 14, 17),  in TEST()
395                                 Token(" ", 17, 18),  in TEST()
396                                 Token("ปร", 18, 20),  in TEST()
397                                 Token(" ", 20, 21),  in TEST()
398                                 Token("มิ", 21, 23)}));  in TEST()
440   std::vector<Token> tokens = tokenizer.Tokenize(  in TEST()
444             std::vector<Token>({Token("こんにちは", 0, 5),  in TEST()
445                                 Token("Japanese-ląnguagę", 5, 22),  in TEST()
446                                 Token("text", 23, 27),  in TEST()
447                                 Token("世界", 28, 30),  in TEST()
448                                 Token("http://www.google.com/", 31, 53)}));  in TEST()
469               std::vector<Token>({Token("앨라배마123웹사이트", 0, 11)}));  in TEST()
478               std::vector<Token>({Token("앨라배마", 0, 4), Token("123", 4, 7),  in TEST()
479                                   Token("웹사이트", 7, 11)}));  in TEST()