Lines Matching defs:tokens
178 std::vector<Token> tokens = tokenizer.Tokenize("Hello world!"); in TEST() local
349 std::vector<Token> tokens; in TEST() local
379 std::vector<Token> tokens = tokenizer.Tokenize("พระบาท สมเด็จ พระ ปร มิ"); in TEST() local
399 std::vector<Token> tokens = in TEST() local
427 std::vector<Token> tokens = tokenizer.Tokenize("3.1 3﹒2 3．3"); in TEST() local
445 std::vector<Token> tokens = tokenizer.Tokenize("พระบาทสมเด็จพระปรมิ"); in TEST() local
495 std::vector<Token> tokens = tokenizer.Tokenize( in TEST() local
548 std::vector<Token> tokens = tokenizer.Tokenize("7% -3.14 68.9#? 7% $99 .18."); in TEST() local
565 std::vector<Token> tokens = tokenizer.Tokenize("２ pércént ３パーセント"); in TEST() local
577 std::vector<Token> tokens = tokenizer.Tokenize("3 3﹒2 3．3%"); in TEST() local
589 std::vector<Token> tokens = tokenizer.Tokenize("15.12.2019 january's 3.2"); in TEST() local
604 std::vector<Token> tokens = tokenizer.Tokenize("The+2345++the +íí+"); in TEST() local
618 std::vector<Token> tokens = tokenizer.Tokenize("2 3 4 5"); in TEST() local