Lines Matching full:token
87 std::vector<Token> tokens{Token("Hělló", 0, 5), in TEST_F()
88 Token("fěěbař@google.com", 6, 23), in TEST_F()
89 Token("heře!", 24, 29)}; in TEST_F()
95 {Token("Hělló", 0, 5), in TEST_F()
96 Token("fěě", 6, 9), in TEST_F()
97 Token("bař", 9, 12), in TEST_F()
98 Token("@google.com", 12, 23), in TEST_F()
99 Token("heře!", 24, 29)})); in TEST_F()
104 std::vector<Token> tokens{Token("Hělló", 0, 5), in TEST_F()
105 Token("fěěbař@google.com", 6, 23), in TEST_F()
106 Token("heře!", 24, 29)}; in TEST_F()
112 {Token("Hělló", 0, 5), in TEST_F()
113 Token("fěěbař", 6, 12), in TEST_F()
114 Token("@google.com", 12, 23), in TEST_F()
115 Token("heře!", 24, 29)})); in TEST_F()
120 std::vector<Token> tokens{Token("Hělló", 0, 5), in TEST_F()
121 Token("fěěbař@google.com", 6, 23), in TEST_F()
122 Token("heře!", 24, 29)}; in TEST_F()
128 {Token("Hělló", 0, 5), in TEST_F()
129 Token("fěě", 6, 9), in TEST_F()
130 Token("bař@google.com", 9, 23), in TEST_F()
131 Token("heře!", 24, 29)})); in TEST_F()
136 std::vector<Token> tokens{Token("Hělló", 0, 5), in TEST_F()
137 Token("fěěbař@google.com", 6, 23), in TEST_F()
138 Token("heře!", 24, 29)}; in TEST_F()
144 {Token("Hělló", 0, 5), in TEST_F()
145 Token("fěěbař@google.com", 6, 23), in TEST_F()
146 Token("heře!", 24, 29)})); in TEST_F()
152 std::vector<Token> tokens{Token("Hělló", 0, 5), in TEST_F()
153 Token("fěěbař@google.com", 6, 23), in TEST_F()
154 Token("heře!", 24, 29)}; in TEST_F()
160 {Token("Hě", 0, 2), in TEST_F()
161 Token("lló", 2, 5), in TEST_F()
162 Token("fěě", 6, 9), in TEST_F()
163 Token("bař@google.com", 9, 23), in TEST_F()
164 Token("heře!", 24, 29)})); in TEST_F()
179 std::vector<Token> tokens = {Token("Fiřst", 0, 5), in TEST_F()
180 Token("Lině", 6, 10), in TEST_F()
181 Token("Sěcond", 11, 17), in TEST_F()
182 Token("Lině", 18, 22), in TEST_F()
183 Token("Thiřd", 23, 28), in TEST_F()
184 Token("Lině", 29, 33)}; in TEST_F()
190 ElementsAreArray({Token("Fiřst", 0, 5), Token("Lině", 6, 10)})); in TEST_F()
204 std::vector<Token> tokens = {Token("Fiřst", 0, 5), in TEST_F()
205 Token("Lině", 6, 10), in TEST_F()
206 Token("Sěcond", 11, 17), in TEST_F()
207 Token("Lině", 18, 22), in TEST_F()
208 Token("Thiřd", 23, 28), in TEST_F()
209 Token("Lině", 29, 33)}; in TEST_F()
215 {Token("Sěcond", 11, 17), Token("Lině", 18, 22)})); in TEST_F()
229 std::vector<Token> tokens = {Token("Fiřst", 0, 5), in TEST_F()
230 Token("Lině", 6, 10), in TEST_F()
231 Token("Sěcond", 11, 17), in TEST_F()
232 Token("Lině", 18, 22), in TEST_F()
233 Token("Thiřd", 23, 28), in TEST_F()
234 Token("Lině", 29, 33)}; in TEST_F()
240 {Token("Thiřd", 23, 28), Token("Lině", 29, 33)})); in TEST_F()
254 std::vector<Token> tokens = {Token("Fiřst", 0, 5), in TEST_F()
255 Token("Lině", 6, 10), in TEST_F()
256 Token("Sěcond", 11, 17), in TEST_F()
257 Token("Lině", 18, 22), in TEST_F()
258 Token("Thiřd", 23, 28), in TEST_F()
259 Token("Lině", 29, 33)}; in TEST_F()
265 {Token("Sěcond", 11, 17), Token("Lině", 18, 22)})); in TEST_F()
281 std::vector<Token> tokens = {Token("Fiřst", 0, 5), in TEST_F()
282 Token("Lině|Sěcond", 6, 17), in TEST_F()
283 Token("Lině", 18, 22), in TEST_F()
284 Token("Thiřd", 23, 28), in TEST_F()
285 Token("Lině", 29, 33)}; in TEST_F()
290 EXPECT_THAT(tokens, ElementsAreArray({Token("Fiřst", 0, 5), in TEST_F()
291 Token("Lině|Sěcond", 6, 17), in TEST_F()
292 Token("Lině", 18, 22)})); in TEST_F()
350 std::vector<Token> tokens = {Token("Fiřst", 0, 5), in TEST_F()
351 Token("Lině", 6, 10), in TEST_F()
352 Token("Sěcond", 18, 23), in TEST_F()
353 Token("Lině", 19, 23), in TEST_F()
354 Token("Thiřd", 23, 28), in TEST_F()
355 Token("Lině", 29, 33)}; in TEST_F()
361 {Token("Fiřst", 0, 5), Token("Lině", 6, 10), in TEST_F()
362 Token("Sěcond", 18, 23), Token("Lině", 19, 23), in TEST_F()
363 Token("Thiřd", 23, 28), Token("Lině", 29, 33)})); in TEST_F()
383 std::vector<Token> tokens = feature_processor.Tokenize("one, two, three"); in TEST_F()
410 // Cross a token boundary. in TEST_F()
457 std::vector<Token> tokens = feature_processor.Tokenize("one, two, three"); in TEST_F()
484 // Cross a token boundary. in TEST_F()
520 {Token("Hělló", 0, 5), Token("world", 6, 11), Token("heře!", 12, 17)}); in TEST_F()
523 // Click is contained in a token. in TEST_F()
526 {Token("Hělló", 0, 5), Token("world", 6, 11), Token("heře!", 12, 17)}); in TEST_F()
532 {Token("Hělló", 0, 5), Token("world", 6, 11), Token("heře!", 12, 17)}); in TEST_F()
542 {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20), in TEST_F()
543 Token("Token4", 21, 27), Token("Token5", 28, 34)}); in TEST_F()
546 // Selection of length 1 token. Exactly aligned indices. in TEST_F()
549 {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20), in TEST_F()
550 Token("Token4", 21, 27), Token("Token5", 28, 34)}); in TEST_F()
553 // Selection marks sub-token range, with no tokens in it. in TEST_F()
556 {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20), in TEST_F()
557 Token("Token4", 21, 27), Token("Token5", 28, 34)}); in TEST_F()
560 // Selection of length 2. Sub-token indices. in TEST_F()
563 {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20), in TEST_F()
564 Token("Token4", 21, 27), Token("Token5", 28, 34)}); in TEST_F()
567 // Selection of length 1. Sub-token indices. in TEST_F()
570 {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20), in TEST_F()
571 Token("Token4", 21, 27), Token("Token5", 28, 34)}); in TEST_F()
659 const std::vector<Token> tokens = {Token("ěěě", 0, 3), Token("řřř", 4, 7), in TEST_F()
660 Token("eee", 8, 11)}; in TEST_F()
708 const std::vector<Token> tokens = {Token("aaa", 0, 3), Token("bbb", 4, 7), in TEST_F()
709 Token("ccc", 8, 11), Token("ddd", 12, 15)}; in TEST_F()
750 const std::vector<Token> tokens = { in TEST_F()
751 Token("aaa", 0, 3), Token("bbb", 4, 7), Token("ccc", 8, 11), in TEST_F()
752 Token("ddd", 12, 15), Token("eee", 16, 19), Token("fff", 20, 23)}; in TEST_F()
796 std::vector<Token> tokens_orig{ in TEST_F()
797 Token("0", 0, 0), Token("1", 0, 0), Token("2", 0, 0), Token("3", 0, 0), in TEST_F()
798 Token("4", 0, 0), Token("5", 0, 0), Token("6", 0, 0), Token("7", 0, 0), in TEST_F()
799 Token("8", 0, 0), Token("9", 0, 0), Token("10", 0, 0), Token("11", 0, 0), in TEST_F()
800 Token("12", 0, 0)}; in TEST_F()
802 std::vector<Token> tokens; in TEST_F()
805 // Try to click first token and see if it gets padded from left. in TEST_F()
810 EXPECT_EQ(tokens, std::vector<Token>({Token(), in TEST_F()
811 Token(), in TEST_F()
812 Token("0", 0, 0), in TEST_F()
813 Token("1", 0, 0), in TEST_F()
814 Token("2", 0, 0)})); in TEST_F()
818 // When we click the second token nothing should get padded. in TEST_F()
823 EXPECT_EQ(tokens, std::vector<Token>({Token("0", 0, 0), in TEST_F()
824 Token("1", 0, 0), in TEST_F()
825 Token("2", 0, 0), in TEST_F()
826 Token("3", 0, 0), in TEST_F()
827 Token("4", 0, 0)})); in TEST_F()
831 // When we click the last token tokens should get padded from the right. in TEST_F()
836 EXPECT_EQ(tokens, std::vector<Token>({Token("10", 0, 0), in TEST_F()
837 Token("11", 0, 0), in TEST_F()
838 Token("12", 0, 0), in TEST_F()
839 Token(), in TEST_F()
840 Token()})); in TEST_F()
846 std::vector<Token> tokens_orig{ in TEST_F()
847 Token("0", 0, 0), Token("1", 0, 0), Token("2", 0, 0), Token("3", 0, 0), in TEST_F()
848 Token("4", 0, 0), Token("5", 0, 0), Token("6", 0, 0), Token("7", 0, 0), in TEST_F()
849 Token("8", 0, 0), Token("9", 0, 0), Token("10", 0, 0), Token("11", 0, 0), in TEST_F()
850 Token("12", 0, 0)}; in TEST_F()
852 std::vector<Token> tokens; in TEST_F()
855 // Try to click first token and see if it gets padded from left to maximum in TEST_F()
861 EXPECT_EQ(tokens, std::vector<Token>({Token(), in TEST_F()
862 Token(), in TEST_F()
863 Token("0", 0, 0), in TEST_F()
864 Token("1", 0, 0), in TEST_F()
865 Token("2", 0, 0), in TEST_F()
866 Token("3", 0, 0), in TEST_F()
867 Token("4", 0, 0), in TEST_F()
868 Token("5", 0, 0)})); in TEST_F()
877 EXPECT_EQ(tokens, std::vector<Token>({Token("1", 0, 0), in TEST_F()
878 Token("2", 0, 0), in TEST_F()
879 Token("3", 0, 0), in TEST_F()
880 Token("4", 0, 0), in TEST_F()
881 Token("5", 0, 0), in TEST_F()
882 Token("6", 0, 0), in TEST_F()
883 Token("7", 0, 0), in TEST_F()
884 Token("8", 0, 0), in TEST_F()
885 Token("9", 0, 0)})); in TEST_F()
894 EXPECT_EQ(tokens, std::vector<Token>({Token("6", 0, 0), in TEST_F()
895 Token("7", 0, 0), in TEST_F()
896 Token("8", 0, 0), in TEST_F()
897 Token("9", 0, 0), in TEST_F()
898 Token("10", 0, 0), in TEST_F()
899 Token("11", 0, 0), in TEST_F()
900 Token("12", 0, 0), in TEST_F()
901 Token(), in TEST_F()
902 Token()})); in TEST_F()
1020 const std::vector<Token> tokens{Token("Hělló", 0, 5), in TEST_F()
1021 Token("fěěbař@google.com", 6, 23), in TEST_F()
1022 Token("heře!", 24, 29)}; in TEST_F()