Lines Matching refs:tokens
26 std::vector<Token> tokens = in TEST() local
29 EXPECT_EQ(tokens.size(), 9); in TEST()
31 EXPECT_EQ(tokens[0].value, "Where"); in TEST()
32 EXPECT_EQ(tokens[0].start, 0); in TEST()
33 EXPECT_EQ(tokens[0].end, 5); in TEST()
35 EXPECT_EQ(tokens[1].value, "is"); in TEST()
36 EXPECT_EQ(tokens[1].start, 6); in TEST()
37 EXPECT_EQ(tokens[1].end, 8); in TEST()
39 EXPECT_EQ(tokens[2].value, "Jörg"); in TEST()
40 EXPECT_EQ(tokens[2].start, 9); in TEST()
41 EXPECT_EQ(tokens[2].end, 13); in TEST()
43 EXPECT_EQ(tokens[3].value, "Borg"); in TEST()
44 EXPECT_EQ(tokens[3].start, 14); in TEST()
45 EXPECT_EQ(tokens[3].end, 18); in TEST()
47 EXPECT_EQ(tokens[4].value, "located?"); in TEST()
48 EXPECT_EQ(tokens[4].start, 19); in TEST()
49 EXPECT_EQ(tokens[4].end, 27); in TEST()
51 EXPECT_EQ(tokens[5].value, "Maybe"); in TEST()
52 EXPECT_EQ(tokens[5].start, 28); in TEST()
53 EXPECT_EQ(tokens[5].end, 33); in TEST()
55 EXPECT_EQ(tokens[6].value, "in"); in TEST()
56 EXPECT_EQ(tokens[6].start, 34); in TEST()
57 EXPECT_EQ(tokens[6].end, 36); in TEST()
59 EXPECT_EQ(tokens[7].value, "Zürich"); in TEST()
60 EXPECT_EQ(tokens[7].start, 37); in TEST()
61 EXPECT_EQ(tokens[7].end, 43); in TEST()
63 EXPECT_EQ(tokens[8].value, "..."); in TEST()
64 EXPECT_EQ(tokens[8].start, 44); in TEST()
65 EXPECT_EQ(tokens[8].end, 47); in TEST()
69 std::vector<Token> tokens = TokenizeOnDelimiters( in TEST() local
72 EXPECT_EQ(tokens.size(), 6); in TEST()
74 EXPECT_EQ(tokens[0].value, "This"); in TEST()
75 EXPECT_EQ(tokens[0].start, 0); in TEST()
76 EXPECT_EQ(tokens[0].end, 4); in TEST()
78 EXPECT_EQ(tokens[1].value, "might"); in TEST()
79 EXPECT_EQ(tokens[1].start, 7); in TEST()
80 EXPECT_EQ(tokens[1].end, 12); in TEST()
82 EXPECT_EQ(tokens[2].value, "be"); in TEST()
83 EXPECT_EQ(tokens[2].start, 13); in TEST()
84 EXPECT_EQ(tokens[2].end, 15); in TEST()
86 EXPECT_EQ(tokens[3].value, "čomplíčateď"); in TEST()
87 EXPECT_EQ(tokens[3].start, 16); in TEST()
88 EXPECT_EQ(tokens[3].end, 27); in TEST()
90 EXPECT_EQ(tokens[4].value, ":"); in TEST()
91 EXPECT_EQ(tokens[4].start, 29); in TEST()
92 EXPECT_EQ(tokens[4].end, 30); in TEST()
94 EXPECT_EQ(tokens[5].value, "Oder"); in TEST()
95 EXPECT_EQ(tokens[5].start, 31); in TEST()
96 EXPECT_EQ(tokens[5].end, 35); in TEST()
100 std::vector<Token> tokens = TokenizeOnDelimiters( in TEST() local
104 EXPECT_EQ(tokens.size(), 9); in TEST()
106 EXPECT_EQ(tokens[0].value, "This"); in TEST()
107 EXPECT_EQ(tokens[0].start, 0); in TEST()
108 EXPECT_EQ(tokens[0].end, 4); in TEST()
110 EXPECT_EQ(tokens[1].value, "might"); in TEST()
111 EXPECT_EQ(tokens[1].start, 7); in TEST()
112 EXPECT_EQ(tokens[1].end, 12); in TEST()
114 EXPECT_EQ(tokens[2].value, "be"); in TEST()
115 EXPECT_EQ(tokens[2].start, 13); in TEST()
116 EXPECT_EQ(tokens[2].end, 15); in TEST()
118 EXPECT_EQ(tokens[3].value, "čomplíčateď"); in TEST()
119 EXPECT_EQ(tokens[3].start, 16); in TEST()
120 EXPECT_EQ(tokens[3].end, 27); in TEST()
122 EXPECT_EQ(tokens[4].value, "?"); in TEST()
123 EXPECT_EQ(tokens[4].start, 27); in TEST()
124 EXPECT_EQ(tokens[4].end, 28); in TEST()
126 EXPECT_EQ(tokens[5].value, "!"); in TEST()
127 EXPECT_EQ(tokens[5].start, 28); in TEST()
128 EXPECT_EQ(tokens[5].end, 29); in TEST()
130 EXPECT_EQ(tokens[6].value, ":"); in TEST()
131 EXPECT_EQ(tokens[6].start, 29); in TEST()
132 EXPECT_EQ(tokens[6].end, 30); in TEST()
134 EXPECT_EQ(tokens[7].value, "Oder"); in TEST()
135 EXPECT_EQ(tokens[7].start, 31); in TEST()
136 EXPECT_EQ(tokens[7].end, 35); in TEST()
138 EXPECT_EQ(tokens[8].value, "?"); in TEST()
139 EXPECT_EQ(tokens[8].start, 35); in TEST()
140 EXPECT_EQ(tokens[8].end, 36); in TEST()
146 std::vector<Token> tokens = in TEST() local
149 EXPECT_THAT(tokens, testing::ElementsAreArray( in TEST()
158 std::vector<Token> tokens = in TEST() local
161 EXPECT_THAT(tokens, in TEST()
169 std::vector<Token> tokens = in TEST() local
172 ASSERT_TRUE(tokens.empty()); in TEST()
178 std::vector<Token> tokens = in TEST() local
181 EXPECT_THAT(tokens, in TEST()
192 std::vector<Token> tokens = in TEST() local
195 EXPECT_THAT(tokens, in TEST()