1 // Copyright 2018 The Amber Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "src/tokenizer.h"
16
17 #include <cmath>
18 #include <limits>
19
20 #include "gtest/gtest.h"
21
22 namespace amber {
23
24 using TokenizerTest = testing::Test;
25
TEST_F(TokenizerTest,ProcessEmpty)26 TEST_F(TokenizerTest, ProcessEmpty) {
27 Tokenizer t("");
28 auto next = t.NextToken();
29 ASSERT_TRUE(next != nullptr);
30 EXPECT_TRUE(next->IsEOS());
31 }
32
TEST_F(TokenizerTest,ProcessString)33 TEST_F(TokenizerTest, ProcessString) {
34 Tokenizer t("TestString");
35 auto next = t.NextToken();
36 ASSERT_TRUE(next != nullptr);
37 EXPECT_TRUE(next->IsString());
38 EXPECT_EQ("TestString", next->AsString());
39
40 next = t.NextToken();
41 ASSERT_TRUE(next != nullptr);
42 EXPECT_TRUE(next->IsEOS());
43 }
44
TEST_F(TokenizerTest,ProcessInt)45 TEST_F(TokenizerTest, ProcessInt) {
46 Tokenizer t("123");
47 auto next = t.NextToken();
48 ASSERT_TRUE(next != nullptr);
49 EXPECT_TRUE(next->IsInteger());
50 EXPECT_EQ(123U, next->AsUint32());
51
52 next = t.NextToken();
53 ASSERT_TRUE(next != nullptr);
54 EXPECT_TRUE(next->IsEOS());
55 }
56
TEST_F(TokenizerTest,ProcessNegative)57 TEST_F(TokenizerTest, ProcessNegative) {
58 Tokenizer t("-123");
59 auto next = t.NextToken();
60 ASSERT_TRUE(next != nullptr);
61 EXPECT_TRUE(next->IsInteger());
62 EXPECT_EQ(-123, next->AsInt32());
63
64 next = t.NextToken();
65 ASSERT_TRUE(next != nullptr);
66 EXPECT_TRUE(next->IsEOS());
67 }
68
TEST_F(TokenizerTest,ProcessDouble)69 TEST_F(TokenizerTest, ProcessDouble) {
70 Tokenizer t("123.456");
71 auto next = t.NextToken();
72 ASSERT_TRUE(next != nullptr);
73 EXPECT_TRUE(next->IsDouble());
74 EXPECT_EQ(123.456f, next->AsFloat());
75
76 next = t.NextToken();
77 ASSERT_TRUE(next != nullptr);
78 EXPECT_TRUE(next->IsEOS());
79 }
80
81 namespace {
82
TestNaN(const std::string & nan_str)83 void TestNaN(const std::string& nan_str) {
84 Tokenizer t(nan_str);
85 auto next = t.NextToken();
86 ASSERT_TRUE(next != nullptr);
87 EXPECT_TRUE(next->IsDouble());
88 EXPECT_TRUE(std::isnan(next->AsDouble()));
89
90 next = t.NextToken();
91 ASSERT_TRUE(next != nullptr);
92 EXPECT_TRUE(next->IsEOS());
93 }
94
95 } // namespace
96
TEST_F(TokenizerTest,ProcessNaN)97 TEST_F(TokenizerTest, ProcessNaN) {
98 TestNaN("nan");
99 TestNaN("naN");
100 TestNaN("nAn");
101 TestNaN("nAN");
102 TestNaN("Nan");
103 TestNaN("NaN");
104 TestNaN("NAn");
105 TestNaN("NAN");
106 }
107
TEST_F(TokenizerTest,ProcessNegativeDouble)108 TEST_F(TokenizerTest, ProcessNegativeDouble) {
109 Tokenizer t("-123.456");
110 auto next = t.NextToken();
111 ASSERT_TRUE(next != nullptr);
112 EXPECT_TRUE(next->IsDouble());
113 EXPECT_EQ(-123.456f, next->AsFloat());
114
115 next = t.NextToken();
116 ASSERT_TRUE(next != nullptr);
117 EXPECT_TRUE(next->IsEOS());
118 }
119
TEST_F(TokenizerTest,ProcessDoubleStartWithDot)120 TEST_F(TokenizerTest, ProcessDoubleStartWithDot) {
121 Tokenizer t(".123456");
122 auto next = t.NextToken();
123 ASSERT_TRUE(next != nullptr);
124 EXPECT_TRUE(next->IsDouble());
125 EXPECT_EQ(.123456f, next->AsFloat());
126
127 next = t.NextToken();
128 ASSERT_TRUE(next != nullptr);
129 EXPECT_TRUE(next->IsEOS());
130 }
131
TEST_F(TokenizerTest,ProcessStringWithNumberInName)132 TEST_F(TokenizerTest, ProcessStringWithNumberInName) {
133 Tokenizer t("BufferAccess32");
134 auto next = t.NextToken();
135 ASSERT_TRUE(next != nullptr);
136 EXPECT_TRUE(next->IsString());
137 EXPECT_EQ("BufferAccess32", next->AsString());
138
139 next = t.NextToken();
140 ASSERT_TRUE(next != nullptr);
141 EXPECT_TRUE(next->IsEOS());
142 }
143
TEST_F(TokenizerTest,ProcessMultiStatement)144 TEST_F(TokenizerTest, ProcessMultiStatement) {
145 Tokenizer t("TestValue 123.456");
146 auto next = t.NextToken();
147 ASSERT_TRUE(next != nullptr);
148 EXPECT_TRUE(next->IsString());
149 EXPECT_EQ("TestValue", next->AsString());
150
151 next = t.NextToken();
152 ASSERT_TRUE(next != nullptr);
153 EXPECT_TRUE(next->IsDouble());
154 EXPECT_EQ(123.456f, next->AsFloat());
155
156 next = t.NextToken();
157 ASSERT_TRUE(next != nullptr);
158 EXPECT_TRUE(next->IsEOS());
159 }
160
TEST_F(TokenizerTest,ProcessMultiLineStatement)161 TEST_F(TokenizerTest, ProcessMultiLineStatement) {
162 Tokenizer t("TestValue 123.456\nAnotherValue\n\nThirdValue 456");
163 auto next = t.NextToken();
164 ASSERT_TRUE(next != nullptr);
165 EXPECT_TRUE(next->IsString());
166 EXPECT_EQ("TestValue", next->AsString());
167 EXPECT_EQ(1U, t.GetCurrentLine());
168
169 next = t.NextToken();
170 ASSERT_TRUE(next != nullptr);
171 EXPECT_TRUE(next->IsDouble());
172 EXPECT_EQ(123.456f, next->AsFloat());
173 EXPECT_EQ(1U, t.GetCurrentLine());
174
175 next = t.NextToken();
176 ASSERT_TRUE(next != nullptr);
177 EXPECT_TRUE(next->IsEOL());
178
179 next = t.NextToken();
180 ASSERT_TRUE(next != nullptr);
181 EXPECT_TRUE(next->IsString());
182 EXPECT_EQ("AnotherValue", next->AsString());
183 EXPECT_EQ(2U, t.GetCurrentLine());
184
185 next = t.NextToken();
186 ASSERT_TRUE(next != nullptr);
187 EXPECT_TRUE(next->IsEOL());
188
189 next = t.NextToken();
190 ASSERT_TRUE(next != nullptr);
191 EXPECT_TRUE(next->IsEOL());
192
193 next = t.NextToken();
194 ASSERT_TRUE(next != nullptr);
195 EXPECT_TRUE(next->IsString());
196 EXPECT_EQ("ThirdValue", next->AsString());
197 EXPECT_EQ(4U, t.GetCurrentLine());
198
199 next = t.NextToken();
200 ASSERT_TRUE(next != nullptr);
201 EXPECT_TRUE(next->IsInteger());
202 EXPECT_EQ(456U, next->AsUint16());
203 EXPECT_EQ(4U, t.GetCurrentLine());
204
205 next = t.NextToken();
206 ASSERT_TRUE(next != nullptr);
207 EXPECT_TRUE(next->IsEOS());
208 }
209
TEST_F(TokenizerTest,ProcessComments)210 TEST_F(TokenizerTest, ProcessComments) {
211 Tokenizer t(R"(# Initial comment string
212 TestValue 123.456
213 AnotherValue # Space before, comment after
214
215 ThirdValue 456)");
216 auto next = t.NextToken();
217 // The comment injects a blank line into the output
218 // so we can handle full line comment and end of line comment the same.
219 ASSERT_TRUE(next != nullptr);
220 EXPECT_TRUE(next->IsEOL());
221
222 next = t.NextToken();
223 ASSERT_TRUE(next != nullptr);
224 EXPECT_TRUE(next->IsString());
225 EXPECT_EQ("TestValue", next->AsString());
226
227 next = t.NextToken();
228 ASSERT_TRUE(next != nullptr);
229 EXPECT_TRUE(next->IsDouble());
230 EXPECT_EQ(123.456f, next->AsFloat());
231
232 next = t.NextToken();
233 ASSERT_TRUE(next != nullptr);
234 EXPECT_TRUE(next->IsEOL());
235
236 next = t.NextToken();
237 ASSERT_TRUE(next != nullptr);
238 EXPECT_TRUE(next->IsString());
239 EXPECT_EQ("AnotherValue", next->AsString());
240
241 next = t.NextToken();
242 ASSERT_TRUE(next != nullptr);
243 EXPECT_TRUE(next->IsEOL());
244
245 next = t.NextToken();
246 ASSERT_TRUE(next != nullptr);
247 EXPECT_TRUE(next->IsEOL());
248
249 next = t.NextToken();
250 ASSERT_TRUE(next != nullptr);
251 EXPECT_TRUE(next->IsString());
252 EXPECT_EQ("ThirdValue", next->AsString());
253
254 next = t.NextToken();
255 ASSERT_TRUE(next != nullptr);
256 EXPECT_TRUE(next->IsInteger());
257 EXPECT_EQ(456U, next->AsUint16());
258
259 next = t.NextToken();
260 ASSERT_TRUE(next != nullptr);
261 EXPECT_TRUE(next->IsEOS());
262 }
263
TEST_F(TokenizerTest,HexValue)264 TEST_F(TokenizerTest, HexValue) {
265 Tokenizer t("0xff00f0ff");
266 auto next = t.NextToken();
267 ASSERT_TRUE(next != nullptr);
268 EXPECT_TRUE(next->IsHex());
269 EXPECT_EQ(0xff00f0ff, next->AsHex());
270
271 next = t.NextToken();
272 ASSERT_TRUE(next != nullptr);
273 EXPECT_TRUE(next->IsEOS());
274 }
275
TEST_F(TokenizerTest,HexValueAfterWhiteSpace)276 TEST_F(TokenizerTest, HexValueAfterWhiteSpace) {
277 Tokenizer t(" \t \t 0xff00f0ff");
278 auto next = t.NextToken();
279 ASSERT_TRUE(next != nullptr);
280 EXPECT_TRUE(next->IsHex());
281 EXPECT_EQ(0xff00f0ff, next->AsHex());
282
283 next = t.NextToken();
284 ASSERT_TRUE(next != nullptr);
285 EXPECT_TRUE(next->IsEOS());
286 }
287
TEST_F(TokenizerTest,StringStartingWithNum)288 TEST_F(TokenizerTest, StringStartingWithNum) {
289 Tokenizer t("1/ABC");
290 auto next = t.NextToken();
291 ASSERT_TRUE(next != nullptr);
292 EXPECT_TRUE(next->IsInteger());
293 EXPECT_EQ(1U, next->AsUint32());
294
295 next = t.NextToken();
296 ASSERT_TRUE(next != nullptr);
297 EXPECT_TRUE(next->IsString());
298 EXPECT_EQ("/ABC", next->AsString());
299 }
300
TEST_F(TokenizerTest,BracketsAndCommas)301 TEST_F(TokenizerTest, BracketsAndCommas) {
302 Tokenizer t("(1.0, 2, abc)");
303 auto next = t.NextToken();
304 ASSERT_TRUE(next != nullptr);
305 EXPECT_TRUE(next->IsOpenBracket());
306
307 next = t.NextToken();
308 ASSERT_TRUE(next != nullptr);
309 EXPECT_TRUE(next->IsDouble());
310 EXPECT_FLOAT_EQ(1.0, next->AsFloat());
311
312 next = t.NextToken();
313 ASSERT_TRUE(next != nullptr);
314 EXPECT_TRUE(next->IsComma());
315
316 next = t.NextToken();
317 ASSERT_TRUE(next != nullptr);
318 EXPECT_TRUE(next->IsInteger());
319 EXPECT_EQ(2U, next->AsUint32());
320
321 next = t.NextToken();
322 ASSERT_TRUE(next != nullptr);
323 EXPECT_TRUE(next->IsComma());
324
325 next = t.NextToken();
326 ASSERT_TRUE(next != nullptr);
327 EXPECT_TRUE(next->IsString());
328 EXPECT_EQ("abc", next->AsString());
329
330 next = t.NextToken();
331 ASSERT_TRUE(next != nullptr);
332 EXPECT_TRUE(next->IsCloseBracket());
333
334 next = t.NextToken();
335 ASSERT_TRUE(next != nullptr);
336 EXPECT_TRUE(next->IsEOS());
337 }
338
TEST_F(TokenizerTest,TokenToDoubleFromDouble)339 TEST_F(TokenizerTest, TokenToDoubleFromDouble) {
340 Tokenizer t("-1.234");
341 auto next = t.NextToken();
342 ASSERT_TRUE(next != nullptr);
343 ASSERT_TRUE(next->IsDouble());
344
345 Result r = next->ConvertToDouble();
346 ASSERT_TRUE(r.IsSuccess());
347 EXPECT_FLOAT_EQ(-1.234f, next->AsFloat());
348 }
349
TEST_F(TokenizerTest,TokenToDoubleFromInt)350 TEST_F(TokenizerTest, TokenToDoubleFromInt) {
351 Tokenizer t("-1");
352 auto next = t.NextToken();
353 ASSERT_TRUE(next != nullptr);
354 ASSERT_TRUE(next->IsInteger());
355
356 Result r = next->ConvertToDouble();
357 ASSERT_TRUE(r.IsSuccess());
358 EXPECT_FLOAT_EQ(-1.0f, next->AsFloat());
359 }
360
TEST_F(TokenizerTest,DashToken)361 TEST_F(TokenizerTest, DashToken) {
362 Tokenizer t("-");
363 auto next = t.NextToken();
364 ASSERT_TRUE(next != nullptr);
365 ASSERT_TRUE(next->IsString());
366 EXPECT_EQ("-", next->AsString());
367 }
368
TEST_F(TokenizerTest,ParseUint64Max)369 TEST_F(TokenizerTest, ParseUint64Max) {
370 Tokenizer t(std::to_string(std::numeric_limits<uint64_t>::max()));
371 auto next = t.NextToken();
372 ASSERT_TRUE(next != nullptr);
373 ASSERT_TRUE(next->IsInteger());
374 EXPECT_EQ(std::numeric_limits<uint64_t>::max(), next->AsUint64());
375 }
376
TEST_F(TokenizerTest,ParseInt64Min)377 TEST_F(TokenizerTest, ParseInt64Min) {
378 Tokenizer t(std::to_string(std::numeric_limits<int64_t>::min()));
379 auto next = t.NextToken();
380 ASSERT_TRUE(next != nullptr);
381 ASSERT_TRUE(next->IsInteger());
382 EXPECT_EQ(std::numeric_limits<int64_t>::min(), next->AsInt64());
383 }
384
TEST_F(TokenizerTest,TokenToDoubleFromUint64Max)385 TEST_F(TokenizerTest, TokenToDoubleFromUint64Max) {
386 Tokenizer t(std::to_string(std::numeric_limits<uint64_t>::max()));
387 auto next = t.NextToken();
388 ASSERT_TRUE(next != nullptr);
389 ASSERT_TRUE(next->IsInteger());
390
391 Result r = next->ConvertToDouble();
392 ASSERT_FALSE(r.IsSuccess());
393 EXPECT_EQ("uint64_t value too big to fit in double", r.Error());
394 }
395
TEST_F(TokenizerTest,TokenToDoubleFromInt64Min)396 TEST_F(TokenizerTest, TokenToDoubleFromInt64Min) {
397 Tokenizer t(std::to_string(std::numeric_limits<int64_t>::min()));
398 auto next = t.NextToken();
399 ASSERT_TRUE(next != nullptr);
400 ASSERT_TRUE(next->IsInteger());
401
402 Result r = next->ConvertToDouble();
403 ASSERT_TRUE(r.IsSuccess());
404 EXPECT_DOUBLE_EQ(static_cast<double>(std::numeric_limits<int64_t>::min()),
405 next->AsDouble());
406 }
407
TEST_F(TokenizerTest,TokenToDoubleFromInt64Max)408 TEST_F(TokenizerTest, TokenToDoubleFromInt64Max) {
409 Tokenizer t(std::to_string(std::numeric_limits<int64_t>::max()));
410 auto next = t.NextToken();
411 ASSERT_TRUE(next != nullptr);
412 ASSERT_TRUE(next->IsInteger());
413
414 Result r = next->ConvertToDouble();
415 ASSERT_TRUE(r.IsSuccess());
416 EXPECT_DOUBLE_EQ(static_cast<double>(std::numeric_limits<int64_t>::max()),
417 next->AsDouble());
418 }
419
TEST_F(TokenizerTest,TokenToDoubleFromString)420 TEST_F(TokenizerTest, TokenToDoubleFromString) {
421 Tokenizer t("INVALID");
422 auto next = t.NextToken();
423 ASSERT_TRUE(next != nullptr);
424 ASSERT_TRUE(next->IsString());
425
426 Result r = next->ConvertToDouble();
427 ASSERT_FALSE(r.IsSuccess());
428 EXPECT_EQ("Invalid conversion to double", r.Error());
429 }
430
TEST_F(TokenizerTest,TokenToDoubleFromHex)431 TEST_F(TokenizerTest, TokenToDoubleFromHex) {
432 Tokenizer t("0xff00f0ff");
433 auto next = t.NextToken();
434 ASSERT_TRUE(next != nullptr);
435 ASSERT_TRUE(next->IsHex());
436
437 Result r = next->ConvertToDouble();
438 ASSERT_TRUE(r.IsSuccess());
439 EXPECT_FLOAT_EQ(static_cast<float>(0xff00f0ff), next->AsFloat());
440 }
441
TEST_F(TokenizerTest,TokenToDoubleFromEOS)442 TEST_F(TokenizerTest, TokenToDoubleFromEOS) {
443 Tokenizer t("");
444 auto next = t.NextToken();
445 ASSERT_TRUE(next != nullptr);
446 ASSERT_TRUE(next->IsEOS());
447
448 Result r = next->ConvertToDouble();
449 ASSERT_FALSE(r.IsSuccess());
450 EXPECT_EQ("Invalid conversion to double", r.Error());
451 }
452
TEST_F(TokenizerTest,TokenToDoubleFromEOL)453 TEST_F(TokenizerTest, TokenToDoubleFromEOL) {
454 Tokenizer t("-1\n-2");
455 auto next = t.NextToken();
456 next = t.NextToken();
457 ASSERT_TRUE(next != nullptr);
458 ASSERT_TRUE(next->IsEOL());
459
460 Result r = next->ConvertToDouble();
461 ASSERT_FALSE(r.IsSuccess());
462 EXPECT_EQ("Invalid conversion to double", r.Error());
463 }
464
TEST_F(TokenizerTest,Continuations)465 TEST_F(TokenizerTest, Continuations) {
466 Tokenizer t("1 \\\n2");
467 auto next = t.NextToken();
468 ASSERT_TRUE(next != nullptr);
469 ASSERT_TRUE(next->IsInteger());
470 EXPECT_EQ(1, next->AsInt32());
471 EXPECT_EQ(1, t.GetCurrentLine());
472
473 next = t.NextToken();
474 ASSERT_TRUE(next != nullptr);
475 ASSERT_TRUE(next->IsInteger());
476 EXPECT_EQ(2, next->AsInt32());
477 EXPECT_EQ(2, t.GetCurrentLine());
478
479 next = t.NextToken();
480 ASSERT_TRUE(next != nullptr);
481 EXPECT_TRUE(next->IsEOS());
482 }
483
TEST_F(TokenizerTest,ContinuationAtEndOfString)484 TEST_F(TokenizerTest, ContinuationAtEndOfString) {
485 Tokenizer t("1 \\");
486 auto next = t.NextToken();
487 ASSERT_TRUE(next != nullptr);
488 ASSERT_TRUE(next->IsInteger());
489 EXPECT_EQ(1, next->AsInt32());
490
491 next = t.NextToken();
492 ASSERT_TRUE(next != nullptr);
493 ASSERT_TRUE(next->IsString());
494 EXPECT_EQ("\\", next->AsString());
495
496 next = t.NextToken();
497 ASSERT_TRUE(next != nullptr);
498 EXPECT_TRUE(next->IsEOS());
499 }
500
TEST_F(TokenizerTest,ContinuationTokenAtOfLine)501 TEST_F(TokenizerTest, ContinuationTokenAtOfLine) {
502 Tokenizer t("1 \\2");
503 auto next = t.NextToken();
504 ASSERT_TRUE(next != nullptr);
505 ASSERT_TRUE(next->IsInteger());
506 EXPECT_EQ(1, next->AsInt32());
507
508 next = t.NextToken();
509 ASSERT_TRUE(next != nullptr);
510 ASSERT_TRUE(next->IsString());
511 EXPECT_EQ("\\2", next->AsString());
512
513 next = t.NextToken();
514 ASSERT_TRUE(next != nullptr);
515 EXPECT_TRUE(next->IsEOS());
516 }
517
TEST_F(TokenizerTest,ContinuationTokenInMiddleOfLine)518 TEST_F(TokenizerTest, ContinuationTokenInMiddleOfLine) {
519 Tokenizer t("1 \\ 2");
520 auto next = t.NextToken();
521 ASSERT_TRUE(next != nullptr);
522 ASSERT_TRUE(next->IsInteger());
523 EXPECT_EQ(1, next->AsInt32());
524
525 next = t.NextToken();
526 ASSERT_TRUE(next != nullptr);
527 ASSERT_TRUE(next->IsString());
528 EXPECT_EQ("\\", next->AsString());
529
530 next = t.NextToken();
531 ASSERT_TRUE(next != nullptr);
532 ASSERT_TRUE(next->IsInteger());
533 EXPECT_EQ(2U, next->AsInt32());
534
535 next = t.NextToken();
536 ASSERT_TRUE(next != nullptr);
537 EXPECT_TRUE(next->IsEOS());
538 }
539
TEST_F(TokenizerTest,ExtractToNext)540 TEST_F(TokenizerTest, ExtractToNext) {
541 Tokenizer t("this\nis\na\ntest\nEND");
542
543 auto next = t.NextToken();
544 EXPECT_TRUE(next->IsString());
545 EXPECT_EQ("this", next->AsString());
546
547 std::string s = t.ExtractToNext("END");
548 ASSERT_EQ("\nis\na\ntest\n", s);
549
550 next = t.NextToken();
551 EXPECT_TRUE(next->IsString());
552 EXPECT_EQ("END", next->AsString());
553 EXPECT_EQ(5U, t.GetCurrentLine());
554
555 next = t.NextToken();
556 EXPECT_TRUE(next->IsEOS());
557 }
558
TEST_F(TokenizerTest,ExtractToNextMissingNext)559 TEST_F(TokenizerTest, ExtractToNextMissingNext) {
560 Tokenizer t("this\nis\na\ntest\n");
561
562 auto next = t.NextToken();
563 EXPECT_TRUE(next->IsString());
564 EXPECT_EQ("this", next->AsString());
565
566 std::string s = t.ExtractToNext("END");
567 ASSERT_EQ("\nis\na\ntest\n", s);
568
569 next = t.NextToken();
570 EXPECT_TRUE(next->IsEOS());
571 EXPECT_EQ(5U, t.GetCurrentLine());
572 }
573
TEST_F(TokenizerTest,ExtractToNextCurrentIsNext)574 TEST_F(TokenizerTest, ExtractToNextCurrentIsNext) {
575 Tokenizer t("END");
576 std::string s = t.ExtractToNext("END");
577 ASSERT_EQ("", s);
578
579 auto next = t.NextToken();
580 EXPECT_TRUE(next->IsString());
581 EXPECT_EQ("END", next->AsString());
582
583 next = t.NextToken();
584 EXPECT_TRUE(next->IsEOS());
585 }
586
587 } // namespace amber
588