1 // Copyright 2020 The Tint Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "src/reader/wgsl/lexer.h"
16
17 #include <limits>
18
19 #include "gtest/gtest.h"
20
21 namespace tint {
22 namespace reader {
23 namespace wgsl {
24 namespace {
25
26 using LexerTest = testing::Test;
27
TEST_F(LexerTest,Empty)28 TEST_F(LexerTest, Empty) {
29 Source::FileContent content("");
30 Lexer l("test.wgsl", &content);
31 auto t = l.next();
32 EXPECT_TRUE(t.IsEof());
33 }
34
TEST_F(LexerTest,Skips_Whitespace)35 TEST_F(LexerTest, Skips_Whitespace) {
36 Source::FileContent content("\t\r\n\t ident\t\n\t \r ");
37 Lexer l("test.wgsl", &content);
38
39 auto t = l.next();
40 EXPECT_TRUE(t.IsIdentifier());
41 EXPECT_EQ(t.source().range.begin.line, 2u);
42 EXPECT_EQ(t.source().range.begin.column, 6u);
43 EXPECT_EQ(t.source().range.end.line, 2u);
44 EXPECT_EQ(t.source().range.end.column, 11u);
45 EXPECT_EQ(t.to_str(), "ident");
46
47 t = l.next();
48 EXPECT_TRUE(t.IsEof());
49 }
50
TEST_F(LexerTest,Skips_Comments_Line)51 TEST_F(LexerTest, Skips_Comments_Line) {
52 Source::FileContent content(R"(//starts with comment
53 ident1 //ends with comment
54 // blank line
55 ident2)");
56 Lexer l("test.wgsl", &content);
57
58 auto t = l.next();
59 EXPECT_TRUE(t.IsIdentifier());
60 EXPECT_EQ(t.source().range.begin.line, 2u);
61 EXPECT_EQ(t.source().range.begin.column, 1u);
62 EXPECT_EQ(t.source().range.end.line, 2u);
63 EXPECT_EQ(t.source().range.end.column, 7u);
64 EXPECT_EQ(t.to_str(), "ident1");
65
66 t = l.next();
67 EXPECT_TRUE(t.IsIdentifier());
68 EXPECT_EQ(t.source().range.begin.line, 4u);
69 EXPECT_EQ(t.source().range.begin.column, 2u);
70 EXPECT_EQ(t.source().range.end.line, 4u);
71 EXPECT_EQ(t.source().range.end.column, 8u);
72 EXPECT_EQ(t.to_str(), "ident2");
73
74 t = l.next();
75 EXPECT_TRUE(t.IsEof());
76 }
77
TEST_F(LexerTest,Skips_Comments_Block)78 TEST_F(LexerTest, Skips_Comments_Block) {
79 Source::FileContent content(R"(/* comment
80 text */ident)");
81 Lexer l("test.wgsl", &content);
82
83 auto t = l.next();
84 EXPECT_TRUE(t.IsIdentifier());
85 EXPECT_EQ(t.source().range.begin.line, 2u);
86 EXPECT_EQ(t.source().range.begin.column, 8u);
87 EXPECT_EQ(t.source().range.end.line, 2u);
88 EXPECT_EQ(t.source().range.end.column, 13u);
89 EXPECT_EQ(t.to_str(), "ident");
90
91 t = l.next();
92 EXPECT_TRUE(t.IsEof());
93 }
94
TEST_F(LexerTest,Skips_Comments_Block_Nested)95 TEST_F(LexerTest, Skips_Comments_Block_Nested) {
96 Source::FileContent content(R"(/* comment
97 text // nested line comments are ignored /* more text
98 /////**/ */*/ident)");
99 Lexer l("test.wgsl", &content);
100
101 auto t = l.next();
102 EXPECT_TRUE(t.IsIdentifier());
103 EXPECT_EQ(t.source().range.begin.line, 3u);
104 EXPECT_EQ(t.source().range.begin.column, 14u);
105 EXPECT_EQ(t.source().range.end.line, 3u);
106 EXPECT_EQ(t.source().range.end.column, 19u);
107 EXPECT_EQ(t.to_str(), "ident");
108
109 t = l.next();
110 EXPECT_TRUE(t.IsEof());
111 }
112
TEST_F(LexerTest,Skips_Comments_Block_Unterminated)113 TEST_F(LexerTest, Skips_Comments_Block_Unterminated) {
114 // I had to break up the /* because otherwise the clang readability check
115 // errored out saying it could not find the end of a multi-line comment.
116 Source::FileContent content(R"(
117 /)"
118 R"(*
119 abcd)");
120 Lexer l("test.wgsl", &content);
121
122 auto t = l.next();
123 ASSERT_TRUE(t.Is(Token::Type::kError));
124 EXPECT_EQ(t.to_str(), "unterminated block comment");
125 EXPECT_EQ(t.source().range.begin.line, 2u);
126 EXPECT_EQ(t.source().range.begin.column, 3u);
127 EXPECT_EQ(t.source().range.end.line, 2u);
128 EXPECT_EQ(t.source().range.end.column, 4u);
129 }
130
131 struct FloatData {
132 const char* input;
133 float result;
134 };
operator <<(std::ostream & out,FloatData data)135 inline std::ostream& operator<<(std::ostream& out, FloatData data) {
136 out << std::string(data.input);
137 return out;
138 }
139 using FloatTest = testing::TestWithParam<FloatData>;
TEST_P(FloatTest,Parse)140 TEST_P(FloatTest, Parse) {
141 auto params = GetParam();
142 Source::FileContent content(params.input);
143 Lexer l("test.wgsl", &content);
144
145 auto t = l.next();
146 EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral));
147 EXPECT_EQ(t.to_f32(), params.result);
148 EXPECT_EQ(t.source().range.begin.line, 1u);
149 EXPECT_EQ(t.source().range.begin.column, 1u);
150 EXPECT_EQ(t.source().range.end.line, 1u);
151 EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
152
153 t = l.next();
154 EXPECT_TRUE(t.IsEof());
155 }
156 INSTANTIATE_TEST_SUITE_P(LexerTest,
157 FloatTest,
158 testing::Values(
159 // No decimal, with 'f' suffix
160 FloatData{"0f", 0.0f},
161 FloatData{"1f", 1.0f},
162 FloatData{"-0f", 0.0f},
163 FloatData{"-1f", -1.0f},
164
165 // Zero, with decimal.
166 FloatData{"0.0", 0.0f},
167 FloatData{"0.", 0.0f},
168 FloatData{".0", 0.0f},
169 FloatData{"-0.0", 0.0f},
170 FloatData{"-0.", 0.0f},
171 FloatData{"-.0", 0.0f},
172 // Zero, with decimal and 'f' suffix
173 FloatData{"0.0f", 0.0f},
174 FloatData{"0.f", 0.0f},
175 FloatData{".0f", 0.0f},
176 FloatData{"-0.0f", 0.0f},
177 FloatData{"-0.f", 0.0f},
178 FloatData{"-.0", 0.0f},
179
180 // Non-zero with decimal
181 FloatData{"5.7", 5.7f},
182 FloatData{"5.", 5.f},
183 FloatData{".7", .7f},
184 FloatData{"-5.7", -5.7f},
185 FloatData{"-5.", -5.f},
186 FloatData{"-.7", -.7f},
187 // Non-zero with decimal and 'f' suffix
188 FloatData{"5.7f", 5.7f},
189 FloatData{"5.f", 5.f},
190 FloatData{".7f", .7f},
191 FloatData{"-5.7f", -5.7f},
192 FloatData{"-5.f", -5.f},
193 FloatData{"-.7f", -.7f},
194
195 // No decimal, with exponent
196 FloatData{"1e5", 1e5f},
197 FloatData{"1E5", 1e5f},
198 FloatData{"1e-5", 1e-5f},
199 FloatData{"1E-5", 1e-5f},
200 // No decimal, with exponent and 'f' suffix
201 FloatData{"1e5f", 1e5f},
202 FloatData{"1E5f", 1e5f},
203 FloatData{"1e-5f", 1e-5f},
204 FloatData{"1E-5f", 1e-5f},
205 // With decimal and exponents
206 FloatData{"0.2e+12", 0.2e12f},
207 FloatData{"1.2e-5", 1.2e-5f},
208 FloatData{"2.57e23", 2.57e23f},
209 FloatData{"2.5e+0", 2.5f},
210 FloatData{"2.5e-0", 2.5f},
211 // With decimal and exponents and 'f' suffix
212 FloatData{"0.2e+12f", 0.2e12f},
213 FloatData{"1.2e-5f", 1.2e-5f},
214 FloatData{"2.57e23f", 2.57e23f},
215 FloatData{"2.5e+0f", 2.5f},
216 FloatData{"2.5e-0f", 2.5f}));
217
218 using FloatTest_Invalid = testing::TestWithParam<const char*>;
TEST_P(FloatTest_Invalid,Handles)219 TEST_P(FloatTest_Invalid, Handles) {
220 Source::FileContent content(GetParam());
221 Lexer l("test.wgsl", &content);
222
223 auto t = l.next();
224 EXPECT_FALSE(t.Is(Token::Type::kFloatLiteral));
225 }
226 INSTANTIATE_TEST_SUITE_P(
227 LexerTest,
228 FloatTest_Invalid,
229 testing::Values(".",
230 "-.",
231 // Need a mantissa digit
232 ".e5",
233 ".E5",
234 // Need exponent digits
235 ".e",
236 ".e+",
237 ".e-",
238 ".E",
239 ".e+",
240 ".e-",
241 // Overflow
242 "2.5e+256",
243 "-2.5e+127",
244 // Magnitude smaller than smallest positive f32.
245 "2.5e-300",
246 "-2.5e-300",
247 // Decimal exponent must immediately
248 // follow the 'e'.
249 "2.5e 12",
250 "2.5e +12",
251 "2.5e -12",
252 "2.5e+ 123",
253 "2.5e- 123",
254 "2.5E 12",
255 "2.5E +12",
256 "2.5E -12",
257 "2.5E+ 123",
258 "2.5E- 123"));
259
260 using IdentifierTest = testing::TestWithParam<const char*>;
TEST_P(IdentifierTest,Parse)261 TEST_P(IdentifierTest, Parse) {
262 Source::FileContent content(GetParam());
263 Lexer l("test.wgsl", &content);
264
265 auto t = l.next();
266 EXPECT_TRUE(t.IsIdentifier());
267 EXPECT_EQ(t.source().range.begin.line, 1u);
268 EXPECT_EQ(t.source().range.begin.column, 1u);
269 EXPECT_EQ(t.source().range.end.line, 1u);
270 EXPECT_EQ(t.source().range.end.column, 1u + strlen(GetParam()));
271 EXPECT_EQ(t.to_str(), GetParam());
272 }
273 INSTANTIATE_TEST_SUITE_P(LexerTest,
274 IdentifierTest,
275 testing::Values("a",
276 "test",
277 "test01",
278 "test_",
279 "_test",
280 "test_01",
281 "ALLCAPS",
282 "MiXeD_CaSe",
283 "abcdefghijklmnopqrstuvwxyz",
284 "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
285 "alldigits_0123456789"));
286
TEST_F(LexerTest,IdentifierTest_SingleUnderscoreDoesNotMatch)287 TEST_F(LexerTest, IdentifierTest_SingleUnderscoreDoesNotMatch) {
288 Source::FileContent content("_");
289 Lexer l("test.wgsl", &content);
290
291 auto t = l.next();
292 EXPECT_FALSE(t.IsIdentifier());
293 }
294
TEST_F(LexerTest,IdentifierTest_DoesNotStartWithDoubleUnderscore)295 TEST_F(LexerTest, IdentifierTest_DoesNotStartWithDoubleUnderscore) {
296 Source::FileContent content("__test");
297 Lexer l("test.wgsl", &content);
298
299 auto t = l.next();
300 EXPECT_FALSE(t.IsIdentifier());
301 }
302
TEST_F(LexerTest,IdentifierTest_DoesNotStartWithNumber)303 TEST_F(LexerTest, IdentifierTest_DoesNotStartWithNumber) {
304 Source::FileContent content("01test");
305 Lexer l("test.wgsl", &content);
306
307 auto t = l.next();
308 EXPECT_FALSE(t.IsIdentifier());
309 }
310
311 struct HexSignedIntData {
312 const char* input;
313 int32_t result;
314 };
operator <<(std::ostream & out,HexSignedIntData data)315 inline std::ostream& operator<<(std::ostream& out, HexSignedIntData data) {
316 out << std::string(data.input);
317 return out;
318 }
319
320 using IntegerTest_HexSigned = testing::TestWithParam<HexSignedIntData>;
TEST_P(IntegerTest_HexSigned,Matches)321 TEST_P(IntegerTest_HexSigned, Matches) {
322 auto params = GetParam();
323 Source::FileContent content(params.input);
324 Lexer l("test.wgsl", &content);
325
326 auto t = l.next();
327 EXPECT_TRUE(t.Is(Token::Type::kSintLiteral));
328 EXPECT_EQ(t.source().range.begin.line, 1u);
329 EXPECT_EQ(t.source().range.begin.column, 1u);
330 EXPECT_EQ(t.source().range.end.line, 1u);
331 EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
332 EXPECT_EQ(t.to_i32(), params.result);
333 }
334 INSTANTIATE_TEST_SUITE_P(
335 LexerTest,
336 IntegerTest_HexSigned,
337 testing::Values(
338 HexSignedIntData{"0x0", 0},
339 HexSignedIntData{"0x42", 66},
340 HexSignedIntData{"-0x42", -66},
341 HexSignedIntData{"0xeF1Abc9", 250719177},
342 HexSignedIntData{"-0x80000000", std::numeric_limits<int32_t>::min()},
343 HexSignedIntData{"0x7FFFFFFF", std::numeric_limits<int32_t>::max()}));
344
TEST_F(LexerTest,IntegerTest_HexSignedTooLarge)345 TEST_F(LexerTest, IntegerTest_HexSignedTooLarge) {
346 Source::FileContent content("0x80000000");
347 Lexer l("test.wgsl", &content);
348
349 auto t = l.next();
350 ASSERT_TRUE(t.Is(Token::Type::kError));
351 EXPECT_EQ(t.to_str(), "i32 (0x80000000) too large");
352 }
353
TEST_F(LexerTest,IntegerTest_HexSignedTooSmall)354 TEST_F(LexerTest, IntegerTest_HexSignedTooSmall) {
355 Source::FileContent content("-0x8000000F");
356 Lexer l("test.wgsl", &content);
357
358 auto t = l.next();
359 ASSERT_TRUE(t.Is(Token::Type::kError));
360 EXPECT_EQ(t.to_str(), "i32 (-0x8000000F) too small");
361 }
362
TEST_F(LexerTest,IntegerTest_HexSignedTooManyDigits)363 TEST_F(LexerTest, IntegerTest_HexSignedTooManyDigits) {
364 {
365 Source::FileContent content("-0x100000000000000000000000");
366 Lexer l("test.wgsl", &content);
367
368 auto t = l.next();
369 ASSERT_TRUE(t.Is(Token::Type::kError));
370 EXPECT_EQ(t.to_str(),
371 "integer literal (-0x10000000...) has too many digits");
372 }
373 {
374 Source::FileContent content("0x100000000000000");
375 Lexer l("test.wgsl", &content);
376
377 auto t = l.next();
378 ASSERT_TRUE(t.Is(Token::Type::kError));
379 EXPECT_EQ(t.to_str(),
380 "integer literal (0x10000000...) has too many digits");
381 }
382 }
383
384 struct HexUnsignedIntData {
385 const char* input;
386 uint32_t result;
387 };
operator <<(std::ostream & out,HexUnsignedIntData data)388 inline std::ostream& operator<<(std::ostream& out, HexUnsignedIntData data) {
389 out << std::string(data.input);
390 return out;
391 }
392 using IntegerTest_HexUnsigned = testing::TestWithParam<HexUnsignedIntData>;
TEST_P(IntegerTest_HexUnsigned,Matches)393 TEST_P(IntegerTest_HexUnsigned, Matches) {
394 auto params = GetParam();
395 Source::FileContent content(params.input);
396 Lexer l("test.wgsl", &content);
397
398 auto t = l.next();
399 EXPECT_TRUE(t.Is(Token::Type::kUintLiteral));
400 EXPECT_EQ(t.source().range.begin.line, 1u);
401 EXPECT_EQ(t.source().range.begin.column, 1u);
402 EXPECT_EQ(t.source().range.end.line, 1u);
403 EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
404 EXPECT_EQ(t.to_u32(), params.result);
405
406 t = l.next();
407 EXPECT_TRUE(t.IsEof());
408 }
409 INSTANTIATE_TEST_SUITE_P(
410 LexerTest,
411 IntegerTest_HexUnsigned,
412 testing::Values(HexUnsignedIntData{"0x0u", 0},
413 HexUnsignedIntData{"0x42u", 66},
414 HexUnsignedIntData{"0xeF1Abc9u", 250719177},
415 HexUnsignedIntData{"0x0u",
416 std::numeric_limits<uint32_t>::min()},
417 HexUnsignedIntData{"0xFFFFFFFFu",
418 std::numeric_limits<uint32_t>::max()}));
419
TEST_F(LexerTest,IntegerTest_HexUnsignedTooManyDigits)420 TEST_F(LexerTest, IntegerTest_HexUnsignedTooManyDigits) {
421 Source::FileContent content("0x1000000000000000000000u");
422 Lexer l("test.wgsl", &content);
423
424 auto t = l.next();
425 ASSERT_TRUE(t.Is(Token::Type::kError));
426 EXPECT_EQ(t.to_str(), "integer literal (0x10000000...) has too many digits");
427 }
428
429 struct UnsignedIntData {
430 const char* input;
431 uint32_t result;
432 };
operator <<(std::ostream & out,UnsignedIntData data)433 inline std::ostream& operator<<(std::ostream& out, UnsignedIntData data) {
434 out << std::string(data.input);
435 return out;
436 }
437 using IntegerTest_Unsigned = testing::TestWithParam<UnsignedIntData>;
TEST_P(IntegerTest_Unsigned,Matches)438 TEST_P(IntegerTest_Unsigned, Matches) {
439 auto params = GetParam();
440 Source::FileContent content(params.input);
441 Lexer l("test.wgsl", &content);
442
443 auto t = l.next();
444 EXPECT_TRUE(t.Is(Token::Type::kUintLiteral));
445 EXPECT_EQ(t.to_u32(), params.result);
446 EXPECT_EQ(t.source().range.begin.line, 1u);
447 EXPECT_EQ(t.source().range.begin.column, 1u);
448 EXPECT_EQ(t.source().range.end.line, 1u);
449 EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
450 }
451 INSTANTIATE_TEST_SUITE_P(LexerTest,
452 IntegerTest_Unsigned,
453 testing::Values(UnsignedIntData{"0u", 0u},
454 UnsignedIntData{"123u", 123u},
455 UnsignedIntData{"4294967295u",
456 4294967295u}));
457
TEST_F(LexerTest,IntegerTest_UnsignedTooManyDigits)458 TEST_F(LexerTest, IntegerTest_UnsignedTooManyDigits) {
459 Source::FileContent content("10000000000000000000000u");
460 Lexer l("test.wgsl", &content);
461
462 auto t = l.next();
463 ASSERT_TRUE(t.Is(Token::Type::kError));
464 EXPECT_EQ(t.to_str(), "integer literal (1000000000...) has too many digits");
465 }
466
467 struct SignedIntData {
468 const char* input;
469 int32_t result;
470 };
operator <<(std::ostream & out,SignedIntData data)471 inline std::ostream& operator<<(std::ostream& out, SignedIntData data) {
472 out << std::string(data.input);
473 return out;
474 }
475 using IntegerTest_Signed = testing::TestWithParam<SignedIntData>;
TEST_P(IntegerTest_Signed,Matches)476 TEST_P(IntegerTest_Signed, Matches) {
477 auto params = GetParam();
478 Source::FileContent content(params.input);
479 Lexer l("test.wgsl", &content);
480
481 auto t = l.next();
482 EXPECT_TRUE(t.Is(Token::Type::kSintLiteral));
483 EXPECT_EQ(t.to_i32(), params.result);
484 EXPECT_EQ(t.source().range.begin.line, 1u);
485 EXPECT_EQ(t.source().range.begin.column, 1u);
486 EXPECT_EQ(t.source().range.end.line, 1u);
487 EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
488 }
489 INSTANTIATE_TEST_SUITE_P(
490 LexerTest,
491 IntegerTest_Signed,
492 testing::Values(SignedIntData{"0", 0},
493 SignedIntData{"-2", -2},
494 SignedIntData{"2", 2},
495 SignedIntData{"123", 123},
496 SignedIntData{"2147483647", 2147483647},
497 SignedIntData{"-2147483648", -2147483648LL}));
498
TEST_F(LexerTest,IntegerTest_SignedTooManyDigits)499 TEST_F(LexerTest, IntegerTest_SignedTooManyDigits) {
500 Source::FileContent content("-10000000000000000");
501 Lexer l("test.wgsl", &content);
502
503 auto t = l.next();
504 ASSERT_TRUE(t.Is(Token::Type::kError));
505 EXPECT_EQ(t.to_str(), "integer literal (-1000000000...) has too many digits");
506 }
507
508 using IntegerTest_Invalid = testing::TestWithParam<const char*>;
TEST_P(IntegerTest_Invalid,Parses)509 TEST_P(IntegerTest_Invalid, Parses) {
510 Source::FileContent content(GetParam());
511 Lexer l("test.wgsl", &content);
512
513 auto t = l.next();
514 EXPECT_FALSE(t.Is(Token::Type::kSintLiteral));
515 EXPECT_FALSE(t.Is(Token::Type::kUintLiteral));
516 }
517 INSTANTIATE_TEST_SUITE_P(LexerTest,
518 IntegerTest_Invalid,
519 testing::Values("2147483648",
520 "4294967296u",
521 "01234",
522 "0000",
523 "-00",
524 "00u"));
525
526 struct TokenData {
527 const char* input;
528 Token::Type type;
529 };
operator <<(std::ostream & out,TokenData data)530 inline std::ostream& operator<<(std::ostream& out, TokenData data) {
531 out << std::string(data.input);
532 return out;
533 }
534 using PunctuationTest = testing::TestWithParam<TokenData>;
TEST_P(PunctuationTest,Parses)535 TEST_P(PunctuationTest, Parses) {
536 auto params = GetParam();
537 Source::FileContent content(params.input);
538 Lexer l("test.wgsl", &content);
539
540 auto t = l.next();
541 EXPECT_TRUE(t.Is(params.type));
542 EXPECT_EQ(t.source().range.begin.line, 1u);
543 EXPECT_EQ(t.source().range.begin.column, 1u);
544 EXPECT_EQ(t.source().range.end.line, 1u);
545 EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
546
547 t = l.next();
548 EXPECT_EQ(t.source().range.begin.column,
549 1 + std::string(params.input).size());
550 }
551 INSTANTIATE_TEST_SUITE_P(
552 LexerTest,
553 PunctuationTest,
554 testing::Values(TokenData{"&", Token::Type::kAnd},
555 TokenData{"&&", Token::Type::kAndAnd},
556 TokenData{"->", Token::Type::kArrow},
557 TokenData{"[[", Token::Type::kAttrLeft},
558 TokenData{"]]", Token::Type::kAttrRight},
559 TokenData{"/", Token::Type::kForwardSlash},
560 TokenData{"!", Token::Type::kBang},
561 TokenData{"[", Token::Type::kBracketLeft},
562 TokenData{"]", Token::Type::kBracketRight},
563 TokenData{"{", Token::Type::kBraceLeft},
564 TokenData{"}", Token::Type::kBraceRight},
565 TokenData{":", Token::Type::kColon},
566 TokenData{",", Token::Type::kComma},
567 TokenData{"=", Token::Type::kEqual},
568 TokenData{"==", Token::Type::kEqualEqual},
569 TokenData{">", Token::Type::kGreaterThan},
570 TokenData{">=", Token::Type::kGreaterThanEqual},
571 TokenData{">>", Token::Type::kShiftRight},
572 TokenData{"<", Token::Type::kLessThan},
573 TokenData{"<=", Token::Type::kLessThanEqual},
574 TokenData{"<<", Token::Type::kShiftLeft},
575 TokenData{"%", Token::Type::kMod},
576 TokenData{"!=", Token::Type::kNotEqual},
577 TokenData{"-", Token::Type::kMinus},
578 TokenData{"--", Token::Type::kMinusMinus},
579 TokenData{".", Token::Type::kPeriod},
580 TokenData{"+", Token::Type::kPlus},
581 TokenData{"++", Token::Type::kPlusPlus},
582 TokenData{"|", Token::Type::kOr},
583 TokenData{"||", Token::Type::kOrOr},
584 TokenData{"(", Token::Type::kParenLeft},
585 TokenData{")", Token::Type::kParenRight},
586 TokenData{";", Token::Type::kSemicolon},
587 TokenData{"*", Token::Type::kStar},
588 TokenData{"~", Token::Type::kTilde},
589 TokenData{"_", Token::Type::kUnderscore},
590 TokenData{"^", Token::Type::kXor}));
591
592 using KeywordTest = testing::TestWithParam<TokenData>;
TEST_P(KeywordTest,Parses)593 TEST_P(KeywordTest, Parses) {
594 auto params = GetParam();
595 Source::FileContent content(params.input);
596 Lexer l("test.wgsl", &content);
597
598 auto t = l.next();
599 EXPECT_TRUE(t.Is(params.type)) << params.input;
600 EXPECT_EQ(t.source().range.begin.line, 1u);
601 EXPECT_EQ(t.source().range.begin.column, 1u);
602 EXPECT_EQ(t.source().range.end.line, 1u);
603 EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
604
605 t = l.next();
606 EXPECT_EQ(t.source().range.begin.column,
607 1 + std::string(params.input).size());
608 }
609 INSTANTIATE_TEST_SUITE_P(
610 LexerTest,
611 KeywordTest,
612 testing::Values(
613 TokenData{"array", Token::Type::kArray},
614 TokenData{"bitcast", Token::Type::kBitcast},
615 TokenData{"bool", Token::Type::kBool},
616 TokenData{"break", Token::Type::kBreak},
617 TokenData{"case", Token::Type::kCase},
618 TokenData{"continue", Token::Type::kContinue},
619 TokenData{"continuing", Token::Type::kContinuing},
620 TokenData{"default", Token::Type::kDefault},
621 TokenData{"discard", Token::Type::kDiscard},
622 TokenData{"else", Token::Type::kElse},
623 TokenData{"elseif", Token::Type::kElseIf},
624 TokenData{"f32", Token::Type::kF32},
625 TokenData{"fallthrough", Token::Type::kFallthrough},
626 TokenData{"false", Token::Type::kFalse},
627 TokenData{"fn", Token::Type::kFn},
628 TokenData{"for", Token::Type::kFor},
629 TokenData{"bgra8unorm", Token::Type::kFormatBgra8Unorm},
630 TokenData{"bgra8unorm_srgb", Token::Type::kFormatBgra8UnormSrgb},
631 TokenData{"r16float", Token::Type::kFormatR16Float},
632 TokenData{"r16sint", Token::Type::kFormatR16Sint},
633 TokenData{"r16uint", Token::Type::kFormatR16Uint},
634 TokenData{"r32float", Token::Type::kFormatR32Float},
635 TokenData{"r32sint", Token::Type::kFormatR32Sint},
636 TokenData{"r32uint", Token::Type::kFormatR32Uint},
637 TokenData{"r8sint", Token::Type::kFormatR8Sint},
638 TokenData{"r8snorm", Token::Type::kFormatR8Snorm},
639 TokenData{"r8uint", Token::Type::kFormatR8Uint},
640 TokenData{"r8unorm", Token::Type::kFormatR8Unorm},
641 TokenData{"rg11b10float", Token::Type::kFormatRg11B10Float},
642 TokenData{"rg16float", Token::Type::kFormatRg16Float},
643 TokenData{"rg16sint", Token::Type::kFormatRg16Sint},
644 TokenData{"rg16uint", Token::Type::kFormatRg16Uint},
645 TokenData{"rg32float", Token::Type::kFormatRg32Float},
646 TokenData{"rg32sint", Token::Type::kFormatRg32Sint},
647 TokenData{"rg32uint", Token::Type::kFormatRg32Uint},
648 TokenData{"rg8sint", Token::Type::kFormatRg8Sint},
649 TokenData{"rg8snorm", Token::Type::kFormatRg8Snorm},
650 TokenData{"rg8uint", Token::Type::kFormatRg8Uint},
651 TokenData{"rg8unorm", Token::Type::kFormatRg8Unorm},
652 TokenData{"rgb10a2unorm", Token::Type::kFormatRgb10A2Unorm},
653 TokenData{"rgba16float", Token::Type::kFormatRgba16Float},
654 TokenData{"rgba16sint", Token::Type::kFormatRgba16Sint},
655 TokenData{"rgba16uint", Token::Type::kFormatRgba16Uint},
656 TokenData{"rgba32float", Token::Type::kFormatRgba32Float},
657 TokenData{"rgba32sint", Token::Type::kFormatRgba32Sint},
658 TokenData{"rgba32uint", Token::Type::kFormatRgba32Uint},
659 TokenData{"rgba8sint", Token::Type::kFormatRgba8Sint},
660 TokenData{"rgba8snorm", Token::Type::kFormatRgba8Snorm},
661 TokenData{"rgba8uint", Token::Type::kFormatRgba8Uint},
662 TokenData{"rgba8unorm", Token::Type::kFormatRgba8Unorm},
663 TokenData{"rgba8unorm_srgb", Token::Type::kFormatRgba8UnormSrgb},
664 TokenData{"function", Token::Type::kFunction},
665 TokenData{"i32", Token::Type::kI32},
666 TokenData{"if", Token::Type::kIf},
667 TokenData{"image", Token::Type::kImage},
668 TokenData{"import", Token::Type::kImport},
669 TokenData{"let", Token::Type::kLet},
670 TokenData{"loop", Token::Type::kLoop},
671 TokenData{"mat2x2", Token::Type::kMat2x2},
672 TokenData{"mat2x3", Token::Type::kMat2x3},
673 TokenData{"mat2x4", Token::Type::kMat2x4},
674 TokenData{"mat3x2", Token::Type::kMat3x2},
675 TokenData{"mat3x3", Token::Type::kMat3x3},
676 TokenData{"mat3x4", Token::Type::kMat3x4},
677 TokenData{"mat4x2", Token::Type::kMat4x2},
678 TokenData{"mat4x3", Token::Type::kMat4x3},
679 TokenData{"mat4x4", Token::Type::kMat4x4},
680 TokenData{"private", Token::Type::kPrivate},
681 TokenData{"ptr", Token::Type::kPtr},
682 TokenData{"return", Token::Type::kReturn},
683 TokenData{"sampler", Token::Type::kSampler},
684 TokenData{"sampler_comparison", Token::Type::kComparisonSampler},
685 TokenData{"storage", Token::Type::kStorage},
686 TokenData{"storage_buffer", Token::Type::kStorage},
687 TokenData{"struct", Token::Type::kStruct},
688 TokenData{"switch", Token::Type::kSwitch},
689 TokenData{"texture_1d", Token::Type::kTextureSampled1d},
690 TokenData{"texture_2d", Token::Type::kTextureSampled2d},
691 TokenData{"texture_2d_array", Token::Type::kTextureSampled2dArray},
692 TokenData{"texture_3d", Token::Type::kTextureSampled3d},
693 TokenData{"texture_cube", Token::Type::kTextureSampledCube},
694 TokenData{"texture_cube_array", Token::Type::kTextureSampledCubeArray},
695 TokenData{"texture_depth_2d", Token::Type::kTextureDepth2d},
696 TokenData{"texture_depth_2d_array", Token::Type::kTextureDepth2dArray},
697 TokenData{"texture_depth_cube", Token::Type::kTextureDepthCube},
698 TokenData{"texture_depth_cube_array",
699 Token::Type::kTextureDepthCubeArray},
700 TokenData{"texture_depth_multisampled_2d",
701 Token::Type::kTextureDepthMultisampled2d},
702 TokenData{"texture_multisampled_2d",
703 Token::Type::kTextureMultisampled2d},
704 TokenData{"texture_storage_1d", Token::Type::kTextureStorage1d},
705 TokenData{"texture_storage_2d", Token::Type::kTextureStorage2d},
706 TokenData{"texture_storage_2d_array",
707 Token::Type::kTextureStorage2dArray},
708 TokenData{"texture_storage_3d", Token::Type::kTextureStorage3d},
709 TokenData{"true", Token::Type::kTrue},
710 TokenData{"type", Token::Type::kType},
711 TokenData{"u32", Token::Type::kU32},
712 TokenData{"uniform", Token::Type::kUniform},
713 TokenData{"var", Token::Type::kVar},
714 TokenData{"vec2", Token::Type::kVec2},
715 TokenData{"vec3", Token::Type::kVec3},
716 TokenData{"vec4", Token::Type::kVec4},
717 TokenData{"workgroup", Token::Type::kWorkgroup}));
718
719 } // namespace
720 } // namespace wgsl
721 } // namespace reader
722 } // namespace tint
723