1 #![allow(
2 clippy::assertions_on_result_states,
3 clippy::items_after_statements,
4 clippy::needless_pass_by_value,
5 clippy::needless_raw_string_hashes,
6 clippy::non_ascii_literal,
7 clippy::octal_escapes
8 )]
9
10 use proc_macro2::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
11 use std::ffi::CStr;
12 use std::iter;
13 use std::str::{self, FromStr};
14
15 #[test]
idents()16 fn idents() {
17 assert_eq!(
18 Ident::new("String", Span::call_site()).to_string(),
19 "String"
20 );
21 assert_eq!(Ident::new("fn", Span::call_site()).to_string(), "fn");
22 assert_eq!(Ident::new("_", Span::call_site()).to_string(), "_");
23 }
24
25 #[test]
raw_idents()26 fn raw_idents() {
27 assert_eq!(
28 Ident::new_raw("String", Span::call_site()).to_string(),
29 "r#String"
30 );
31 assert_eq!(Ident::new_raw("fn", Span::call_site()).to_string(), "r#fn");
32 }
33
34 #[test]
35 #[should_panic(expected = "`r#_` cannot be a raw identifier")]
ident_raw_underscore()36 fn ident_raw_underscore() {
37 Ident::new_raw("_", Span::call_site());
38 }
39
40 #[test]
41 #[should_panic(expected = "`r#super` cannot be a raw identifier")]
ident_raw_reserved()42 fn ident_raw_reserved() {
43 Ident::new_raw("super", Span::call_site());
44 }
45
46 #[test]
47 #[should_panic(expected = "Ident is not allowed to be empty; use Option<Ident>")]
ident_empty()48 fn ident_empty() {
49 Ident::new("", Span::call_site());
50 }
51
52 #[test]
53 #[should_panic(expected = "Ident cannot be a number; use Literal instead")]
ident_number()54 fn ident_number() {
55 Ident::new("255", Span::call_site());
56 }
57
58 #[test]
59 #[should_panic(expected = "\"a#\" is not a valid Ident")]
ident_invalid()60 fn ident_invalid() {
61 Ident::new("a#", Span::call_site());
62 }
63
64 #[test]
65 #[should_panic(expected = "not a valid Ident")]
raw_ident_empty()66 fn raw_ident_empty() {
67 Ident::new("r#", Span::call_site());
68 }
69
70 #[test]
71 #[should_panic(expected = "not a valid Ident")]
raw_ident_number()72 fn raw_ident_number() {
73 Ident::new("r#255", Span::call_site());
74 }
75
76 #[test]
77 #[should_panic(expected = "\"r#a#\" is not a valid Ident")]
raw_ident_invalid()78 fn raw_ident_invalid() {
79 Ident::new("r#a#", Span::call_site());
80 }
81
82 #[test]
83 #[should_panic(expected = "not a valid Ident")]
lifetime_empty()84 fn lifetime_empty() {
85 Ident::new("'", Span::call_site());
86 }
87
88 #[test]
89 #[should_panic(expected = "not a valid Ident")]
lifetime_number()90 fn lifetime_number() {
91 Ident::new("'255", Span::call_site());
92 }
93
94 #[test]
95 #[should_panic(expected = r#""'a#" is not a valid Ident"#)]
lifetime_invalid()96 fn lifetime_invalid() {
97 Ident::new("'a#", Span::call_site());
98 }
99
100 #[test]
literal_string()101 fn literal_string() {
102 #[track_caller]
103 fn assert(literal: Literal, expected: &str) {
104 assert_eq!(literal.to_string(), expected.trim());
105 }
106
107 assert(Literal::string(""), r#" "" "#);
108 assert(Literal::string("aA"), r#" "aA" "#);
109 assert(Literal::string("\t"), r#" "\t" "#);
110 assert(Literal::string("❤"), r#" "❤" "#);
111 assert(Literal::string("'"), r#" "'" "#);
112 assert(Literal::string("\""), r#" "\"" "#);
113 assert(Literal::string("\0"), r#" "\0" "#);
114 assert(Literal::string("\u{1}"), r#" "\u{1}" "#);
115 assert(
116 Literal::string("a\00b\07c\08d\0e\0"),
117 r#" "a\x000b\x007c\08d\0e\0" "#,
118 );
119
120 "\"\\\r\n x\"".parse::<TokenStream>().unwrap();
121 "\"\\\r\n \rx\"".parse::<TokenStream>().unwrap_err();
122 }
123
124 #[test]
literal_raw_string()125 fn literal_raw_string() {
126 "r\"\r\n\"".parse::<TokenStream>().unwrap();
127
128 fn raw_string_literal_with_hashes(n: usize) -> String {
129 let mut literal = String::new();
130 literal.push('r');
131 literal.extend(iter::repeat('#').take(n));
132 literal.push('"');
133 literal.push('"');
134 literal.extend(iter::repeat('#').take(n));
135 literal
136 }
137
138 raw_string_literal_with_hashes(255)
139 .parse::<TokenStream>()
140 .unwrap();
141
142 // https://github.com/rust-lang/rust/pull/95251
143 raw_string_literal_with_hashes(256)
144 .parse::<TokenStream>()
145 .unwrap_err();
146 }
147
148 #[test]
literal_byte_character()149 fn literal_byte_character() {
150 #[track_caller]
151 fn assert(literal: Literal, expected: &str) {
152 assert_eq!(literal.to_string(), expected.trim());
153 }
154
155 assert(Literal::byte_character(b'a'), r#" b'a' "#);
156 assert(Literal::byte_character(b'\0'), r#" b'\0' "#);
157 assert(Literal::byte_character(b'\t'), r#" b'\t' "#);
158 assert(Literal::byte_character(b'\n'), r#" b'\n' "#);
159 assert(Literal::byte_character(b'\r'), r#" b'\r' "#);
160 assert(Literal::byte_character(b'\''), r#" b'\'' "#);
161 assert(Literal::byte_character(b'\\'), r#" b'\\' "#);
162 assert(Literal::byte_character(b'\x1f'), r#" b'\x1F' "#);
163 assert(Literal::byte_character(b'"'), r#" b'"' "#);
164 }
165
166 #[test]
literal_byte_string()167 fn literal_byte_string() {
168 #[track_caller]
169 fn assert(literal: Literal, expected: &str) {
170 assert_eq!(literal.to_string(), expected.trim());
171 }
172
173 assert(Literal::byte_string(b""), r#" b"" "#);
174 assert(Literal::byte_string(b"\0"), r#" b"\0" "#);
175 assert(Literal::byte_string(b"\t"), r#" b"\t" "#);
176 assert(Literal::byte_string(b"\n"), r#" b"\n" "#);
177 assert(Literal::byte_string(b"\r"), r#" b"\r" "#);
178 assert(Literal::byte_string(b"\""), r#" b"\"" "#);
179 assert(Literal::byte_string(b"\\"), r#" b"\\" "#);
180 assert(Literal::byte_string(b"\x1f"), r#" b"\x1F" "#);
181 assert(Literal::byte_string(b"'"), r#" b"'" "#);
182 assert(
183 Literal::byte_string(b"a\00b\07c\08d\0e\0"),
184 r#" b"a\x000b\x007c\08d\0e\0" "#,
185 );
186
187 "b\"\\\r\n x\"".parse::<TokenStream>().unwrap();
188 "b\"\\\r\n \rx\"".parse::<TokenStream>().unwrap_err();
189 "b\"\\\r\n \u{a0}x\"".parse::<TokenStream>().unwrap_err();
190 "br\"\u{a0}\"".parse::<TokenStream>().unwrap_err();
191 }
192
193 #[test]
literal_c_string()194 fn literal_c_string() {
195 #[track_caller]
196 fn assert(literal: Literal, expected: &str) {
197 assert_eq!(literal.to_string(), expected.trim());
198 }
199
200 assert(Literal::c_string(<&CStr>::default()), r#" c"" "#);
201 assert(
202 Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()),
203 r#" c"aA" "#,
204 );
205 assert(
206 Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()),
207 r#" c"aA" "#,
208 );
209 assert(
210 Literal::c_string(CStr::from_bytes_with_nul(b"\t\0").unwrap()),
211 r#" c"\t" "#,
212 );
213 assert(
214 Literal::c_string(CStr::from_bytes_with_nul(b"\xE2\x9D\xA4\0").unwrap()),
215 r#" c"❤" "#,
216 );
217 assert(
218 Literal::c_string(CStr::from_bytes_with_nul(b"'\0").unwrap()),
219 r#" c"'" "#,
220 );
221 assert(
222 Literal::c_string(CStr::from_bytes_with_nul(b"\"\0").unwrap()),
223 r#" c"\"" "#,
224 );
225 assert(
226 Literal::c_string(CStr::from_bytes_with_nul(b"\x7F\xFF\xFE\xCC\xB3\0").unwrap()),
227 r#" c"\u{7f}\xFF\xFE\u{333}" "#,
228 );
229
230 let strings = r###"
231 c"hello\x80我叫\u{1F980}" // from the RFC
232 cr"\"
233 cr##"Hello "world"!"##
234 c"\t\n\r\"\\"
235 "###;
236
237 let mut tokens = strings.parse::<TokenStream>().unwrap().into_iter();
238
239 for expected in &[
240 r#"c"hello\x80我叫\u{1F980}""#,
241 r#"cr"\""#,
242 r###"cr##"Hello "world"!"##"###,
243 r#"c"\t\n\r\"\\""#,
244 ] {
245 match tokens.next().unwrap() {
246 TokenTree::Literal(literal) => {
247 assert_eq!(literal.to_string(), *expected);
248 }
249 unexpected => panic!("unexpected token: {:?}", unexpected),
250 }
251 }
252
253 if let Some(unexpected) = tokens.next() {
254 panic!("unexpected token: {:?}", unexpected);
255 }
256
257 for invalid in &[r#"c"\0""#, r#"c"\x00""#, r#"c"\u{0}""#, "c\"\0\""] {
258 if let Ok(unexpected) = invalid.parse::<TokenStream>() {
259 panic!("unexpected token: {:?}", unexpected);
260 }
261 }
262 }
263
264 #[test]
literal_character()265 fn literal_character() {
266 #[track_caller]
267 fn assert(literal: Literal, expected: &str) {
268 assert_eq!(literal.to_string(), expected.trim());
269 }
270
271 assert(Literal::character('a'), r#" 'a' "#);
272 assert(Literal::character('\t'), r#" '\t' "#);
273 assert(Literal::character('❤'), r#" '❤' "#);
274 assert(Literal::character('\''), r#" '\'' "#);
275 assert(Literal::character('"'), r#" '"' "#);
276 assert(Literal::character('\0'), r#" '\0' "#);
277 assert(Literal::character('\u{1}'), r#" '\u{1}' "#);
278 }
279
280 #[test]
literal_integer()281 fn literal_integer() {
282 #[track_caller]
283 fn assert(literal: Literal, expected: &str) {
284 assert_eq!(literal.to_string(), expected);
285 }
286
287 assert(Literal::u8_suffixed(10), "10u8");
288 assert(Literal::u16_suffixed(10), "10u16");
289 assert(Literal::u32_suffixed(10), "10u32");
290 assert(Literal::u64_suffixed(10), "10u64");
291 assert(Literal::u128_suffixed(10), "10u128");
292 assert(Literal::usize_suffixed(10), "10usize");
293
294 assert(Literal::i8_suffixed(10), "10i8");
295 assert(Literal::i16_suffixed(10), "10i16");
296 assert(Literal::i32_suffixed(10), "10i32");
297 assert(Literal::i64_suffixed(10), "10i64");
298 assert(Literal::i128_suffixed(10), "10i128");
299 assert(Literal::isize_suffixed(10), "10isize");
300
301 assert(Literal::u8_unsuffixed(10), "10");
302 assert(Literal::u16_unsuffixed(10), "10");
303 assert(Literal::u32_unsuffixed(10), "10");
304 assert(Literal::u64_unsuffixed(10), "10");
305 assert(Literal::u128_unsuffixed(10), "10");
306 assert(Literal::usize_unsuffixed(10), "10");
307
308 assert(Literal::i8_unsuffixed(10), "10");
309 assert(Literal::i16_unsuffixed(10), "10");
310 assert(Literal::i32_unsuffixed(10), "10");
311 assert(Literal::i64_unsuffixed(10), "10");
312 assert(Literal::i128_unsuffixed(10), "10");
313 assert(Literal::isize_unsuffixed(10), "10");
314
315 assert(Literal::i32_suffixed(-10), "-10i32");
316 assert(Literal::i32_unsuffixed(-10), "-10");
317 }
318
319 #[test]
literal_float()320 fn literal_float() {
321 #[track_caller]
322 fn assert(literal: Literal, expected: &str) {
323 assert_eq!(literal.to_string(), expected);
324 }
325
326 assert(Literal::f32_suffixed(10.0), "10f32");
327 assert(Literal::f32_suffixed(-10.0), "-10f32");
328 assert(Literal::f64_suffixed(10.0), "10f64");
329 assert(Literal::f64_suffixed(-10.0), "-10f64");
330
331 assert(Literal::f32_unsuffixed(10.0), "10.0");
332 assert(Literal::f32_unsuffixed(-10.0), "-10.0");
333 assert(Literal::f64_unsuffixed(10.0), "10.0");
334 assert(Literal::f64_unsuffixed(-10.0), "-10.0");
335
336 assert(
337 Literal::f64_unsuffixed(1e100),
338 "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0",
339 );
340 }
341
342 #[test]
literal_suffix()343 fn literal_suffix() {
344 fn token_count(p: &str) -> usize {
345 p.parse::<TokenStream>().unwrap().into_iter().count()
346 }
347
348 assert_eq!(token_count("999u256"), 1);
349 assert_eq!(token_count("999r#u256"), 3);
350 assert_eq!(token_count("1."), 1);
351 assert_eq!(token_count("1.f32"), 3);
352 assert_eq!(token_count("1.0_0"), 1);
353 assert_eq!(token_count("1._0"), 3);
354 assert_eq!(token_count("1._m"), 3);
355 assert_eq!(token_count("\"\"s"), 1);
356 assert_eq!(token_count("r\"\"r"), 1);
357 assert_eq!(token_count("r#\"\"#r"), 1);
358 assert_eq!(token_count("b\"\"b"), 1);
359 assert_eq!(token_count("br\"\"br"), 1);
360 assert_eq!(token_count("br#\"\"#br"), 1);
361 assert_eq!(token_count("c\"\"c"), 1);
362 assert_eq!(token_count("cr\"\"cr"), 1);
363 assert_eq!(token_count("cr#\"\"#cr"), 1);
364 assert_eq!(token_count("'c'c"), 1);
365 assert_eq!(token_count("b'b'b"), 1);
366 assert_eq!(token_count("0E"), 1);
367 assert_eq!(token_count("0o0A"), 1);
368 assert_eq!(token_count("0E--0"), 4);
369 assert_eq!(token_count("0.0ECMA"), 1);
370 }
371
372 #[test]
literal_iter_negative()373 fn literal_iter_negative() {
374 let negative_literal = Literal::i32_suffixed(-3);
375 let tokens = TokenStream::from(TokenTree::Literal(negative_literal));
376 let mut iter = tokens.into_iter();
377 match iter.next().unwrap() {
378 TokenTree::Punct(punct) => {
379 assert_eq!(punct.as_char(), '-');
380 assert_eq!(punct.spacing(), Spacing::Alone);
381 }
382 unexpected => panic!("unexpected token {:?}", unexpected),
383 }
384 match iter.next().unwrap() {
385 TokenTree::Literal(literal) => {
386 assert_eq!(literal.to_string(), "3i32");
387 }
388 unexpected => panic!("unexpected token {:?}", unexpected),
389 }
390 assert!(iter.next().is_none());
391 }
392
393 #[test]
literal_parse()394 fn literal_parse() {
395 assert!("1".parse::<Literal>().is_ok());
396 assert!("-1".parse::<Literal>().is_ok());
397 assert!("-1u12".parse::<Literal>().is_ok());
398 assert!("1.0".parse::<Literal>().is_ok());
399 assert!("-1.0".parse::<Literal>().is_ok());
400 assert!("-1.0f12".parse::<Literal>().is_ok());
401 assert!("'a'".parse::<Literal>().is_ok());
402 assert!("\"\n\"".parse::<Literal>().is_ok());
403 assert!("0 1".parse::<Literal>().is_err());
404 assert!(" 0".parse::<Literal>().is_err());
405 assert!("0 ".parse::<Literal>().is_err());
406 assert!("/* comment */0".parse::<Literal>().is_err());
407 assert!("0/* comment */".parse::<Literal>().is_err());
408 assert!("0// comment".parse::<Literal>().is_err());
409 assert!("- 1".parse::<Literal>().is_err());
410 assert!("- 1.0".parse::<Literal>().is_err());
411 assert!("-\"\"".parse::<Literal>().is_err());
412 }
413
414 #[test]
literal_span()415 fn literal_span() {
416 let positive = "0.1".parse::<Literal>().unwrap();
417 let negative = "-0.1".parse::<Literal>().unwrap();
418 let subspan = positive.subspan(1..2);
419
420 #[cfg(not(span_locations))]
421 {
422 let _ = negative;
423 assert!(subspan.is_none());
424 }
425
426 #[cfg(span_locations)]
427 {
428 assert_eq!(positive.span().start().column, 0);
429 assert_eq!(positive.span().end().column, 3);
430 assert_eq!(negative.span().start().column, 0);
431 assert_eq!(negative.span().end().column, 4);
432 assert_eq!(subspan.unwrap().source_text().unwrap(), ".");
433 }
434
435 assert!(positive.subspan(1..4).is_none());
436 }
437
438 #[cfg(span_locations)]
439 #[test]
source_text()440 fn source_text() {
441 let input = " �� a z ";
442 let mut tokens = input
443 .parse::<proc_macro2::TokenStream>()
444 .unwrap()
445 .into_iter();
446
447 let first = tokens.next().unwrap();
448 assert_eq!("��", first.span().source_text().unwrap());
449
450 let second = tokens.next().unwrap();
451 let third = tokens.next().unwrap();
452 assert_eq!("z", third.span().source_text().unwrap());
453 assert_eq!("a", second.span().source_text().unwrap());
454 }
455
456 #[test]
roundtrip()457 fn roundtrip() {
458 fn roundtrip(p: &str) {
459 println!("parse: {}", p);
460 let s = p.parse::<TokenStream>().unwrap().to_string();
461 println!("first: {}", s);
462 let s2 = s.parse::<TokenStream>().unwrap().to_string();
463 assert_eq!(s, s2);
464 }
465 roundtrip("a");
466 roundtrip("<<");
467 roundtrip("<<=");
468 roundtrip(
469 "
470 1
471 1.0
472 1f32
473 2f64
474 1usize
475 4isize
476 4e10
477 1_000
478 1_0i32
479 8u8
480 9
481 0
482 0xffffffffffffffffffffffffffffffff
483 1x
484 1u80
485 1f320
486 ",
487 );
488 roundtrip("'a");
489 roundtrip("'_");
490 roundtrip("'static");
491 roundtrip(r"'\u{10__FFFF}'");
492 roundtrip("\"\\u{10_F0FF__}foo\\u{1_0_0_0__}\"");
493 }
494
495 #[test]
fail()496 fn fail() {
497 fn fail(p: &str) {
498 if let Ok(s) = p.parse::<TokenStream>() {
499 panic!("should have failed to parse: {}\n{:#?}", p, s);
500 }
501 }
502 fail("' static");
503 fail("r#1");
504 fail("r#_");
505 fail("\"\\u{0000000}\""); // overlong unicode escape (rust allows at most 6 hex digits)
506 fail("\"\\u{999999}\""); // outside of valid range of char
507 fail("\"\\u{_0}\""); // leading underscore
508 fail("\"\\u{}\""); // empty
509 fail("b\"\r\""); // bare carriage return in byte string
510 fail("r\"\r\""); // bare carriage return in raw string
511 fail("\"\\\r \""); // backslash carriage return
512 fail("'aa'aa");
513 fail("br##\"\"#");
514 fail("cr##\"\"#");
515 fail("\"\\\n\u{85}\r\"");
516 }
517
518 #[cfg(span_locations)]
519 #[test]
span_test()520 fn span_test() {
521 check_spans(
522 "\
523 /// This is a document comment
524 testing 123
525 {
526 testing 234
527 }",
528 &[
529 (1, 0, 1, 30), // #
530 (1, 0, 1, 30), // [ ... ]
531 (1, 0, 1, 30), // doc
532 (1, 0, 1, 30), // =
533 (1, 0, 1, 30), // "This is..."
534 (2, 0, 2, 7), // testing
535 (2, 8, 2, 11), // 123
536 (3, 0, 5, 1), // { ... }
537 (4, 2, 4, 9), // testing
538 (4, 10, 4, 13), // 234
539 ],
540 );
541 }
542
543 #[cfg(procmacro2_semver_exempt)]
544 #[test]
default_span()545 fn default_span() {
546 let start = Span::call_site().start();
547 assert_eq!(start.line, 1);
548 assert_eq!(start.column, 0);
549 let end = Span::call_site().end();
550 assert_eq!(end.line, 1);
551 assert_eq!(end.column, 0);
552 let source_file = Span::call_site().source_file();
553 assert_eq!(source_file.path().to_string_lossy(), "<unspecified>");
554 assert!(!source_file.is_real());
555 }
556
557 #[cfg(procmacro2_semver_exempt)]
558 #[test]
span_join()559 fn span_join() {
560 let source1 = "aaa\nbbb"
561 .parse::<TokenStream>()
562 .unwrap()
563 .into_iter()
564 .collect::<Vec<_>>();
565 let source2 = "ccc\nddd"
566 .parse::<TokenStream>()
567 .unwrap()
568 .into_iter()
569 .collect::<Vec<_>>();
570
571 assert!(source1[0].span().source_file() != source2[0].span().source_file());
572 assert_eq!(
573 source1[0].span().source_file(),
574 source1[1].span().source_file()
575 );
576
577 let joined1 = source1[0].span().join(source1[1].span());
578 let joined2 = source1[0].span().join(source2[0].span());
579 assert!(joined1.is_some());
580 assert!(joined2.is_none());
581
582 let start = joined1.unwrap().start();
583 let end = joined1.unwrap().end();
584 assert_eq!(start.line, 1);
585 assert_eq!(start.column, 0);
586 assert_eq!(end.line, 2);
587 assert_eq!(end.column, 3);
588
589 assert_eq!(
590 joined1.unwrap().source_file(),
591 source1[0].span().source_file()
592 );
593 }
594
595 #[test]
no_panic()596 fn no_panic() {
597 let s = str::from_utf8(b"b\'\xc2\x86 \x00\x00\x00^\"").unwrap();
598 assert!(s.parse::<TokenStream>().is_err());
599 }
600
601 #[test]
punct_before_comment()602 fn punct_before_comment() {
603 let mut tts = TokenStream::from_str("~// comment").unwrap().into_iter();
604 match tts.next().unwrap() {
605 TokenTree::Punct(tt) => {
606 assert_eq!(tt.as_char(), '~');
607 assert_eq!(tt.spacing(), Spacing::Alone);
608 }
609 wrong => panic!("wrong token {:?}", wrong),
610 }
611 }
612
613 #[test]
joint_last_token()614 fn joint_last_token() {
615 // This test verifies that we match the behavior of libproc_macro *not* in
616 // the range nightly-2020-09-06 through nightly-2020-09-10, in which this
617 // behavior was temporarily broken.
618 // See https://github.com/rust-lang/rust/issues/76399
619
620 let joint_punct = Punct::new(':', Spacing::Joint);
621 let stream = TokenStream::from(TokenTree::Punct(joint_punct));
622 let punct = match stream.into_iter().next().unwrap() {
623 TokenTree::Punct(punct) => punct,
624 _ => unreachable!(),
625 };
626 assert_eq!(punct.spacing(), Spacing::Joint);
627 }
628
629 #[test]
raw_identifier()630 fn raw_identifier() {
631 let mut tts = TokenStream::from_str("r#dyn").unwrap().into_iter();
632 match tts.next().unwrap() {
633 TokenTree::Ident(raw) => assert_eq!("r#dyn", raw.to_string()),
634 wrong => panic!("wrong token {:?}", wrong),
635 }
636 assert!(tts.next().is_none());
637 }
638
639 #[test]
test_debug_ident()640 fn test_debug_ident() {
641 let ident = Ident::new("proc_macro", Span::call_site());
642
643 #[cfg(not(span_locations))]
644 let expected = "Ident(proc_macro)";
645
646 #[cfg(span_locations)]
647 let expected = "Ident { sym: proc_macro }";
648
649 assert_eq!(expected, format!("{:?}", ident));
650 }
651
652 #[test]
test_debug_tokenstream()653 fn test_debug_tokenstream() {
654 let tts = TokenStream::from_str("[a + 1]").unwrap();
655
656 #[cfg(not(span_locations))]
657 let expected = "\
658 TokenStream [
659 Group {
660 delimiter: Bracket,
661 stream: TokenStream [
662 Ident {
663 sym: a,
664 },
665 Punct {
666 char: '+',
667 spacing: Alone,
668 },
669 Literal {
670 lit: 1,
671 },
672 ],
673 },
674 ]\
675 ";
676
677 #[cfg(not(span_locations))]
678 let expected_before_trailing_commas = "\
679 TokenStream [
680 Group {
681 delimiter: Bracket,
682 stream: TokenStream [
683 Ident {
684 sym: a
685 },
686 Punct {
687 char: '+',
688 spacing: Alone
689 },
690 Literal {
691 lit: 1
692 }
693 ]
694 }
695 ]\
696 ";
697
698 #[cfg(span_locations)]
699 let expected = "\
700 TokenStream [
701 Group {
702 delimiter: Bracket,
703 stream: TokenStream [
704 Ident {
705 sym: a,
706 span: bytes(2..3),
707 },
708 Punct {
709 char: '+',
710 spacing: Alone,
711 span: bytes(4..5),
712 },
713 Literal {
714 lit: 1,
715 span: bytes(6..7),
716 },
717 ],
718 span: bytes(1..8),
719 },
720 ]\
721 ";
722
723 #[cfg(span_locations)]
724 let expected_before_trailing_commas = "\
725 TokenStream [
726 Group {
727 delimiter: Bracket,
728 stream: TokenStream [
729 Ident {
730 sym: a,
731 span: bytes(2..3)
732 },
733 Punct {
734 char: '+',
735 spacing: Alone,
736 span: bytes(4..5)
737 },
738 Literal {
739 lit: 1,
740 span: bytes(6..7)
741 }
742 ],
743 span: bytes(1..8)
744 }
745 ]\
746 ";
747
748 let actual = format!("{:#?}", tts);
749 if actual.ends_with(",\n]") {
750 assert_eq!(expected, actual);
751 } else {
752 assert_eq!(expected_before_trailing_commas, actual);
753 }
754 }
755
756 #[test]
default_tokenstream_is_empty()757 fn default_tokenstream_is_empty() {
758 let default_token_stream = <TokenStream as Default>::default();
759
760 assert!(default_token_stream.is_empty());
761 }
762
763 #[test]
tokenstream_size_hint()764 fn tokenstream_size_hint() {
765 let tokens = "a b (c d) e".parse::<TokenStream>().unwrap();
766
767 assert_eq!(tokens.into_iter().size_hint(), (4, Some(4)));
768 }
769
770 #[test]
tuple_indexing()771 fn tuple_indexing() {
772 // This behavior may change depending on https://github.com/rust-lang/rust/pull/71322
773 let mut tokens = "tuple.0.0".parse::<TokenStream>().unwrap().into_iter();
774 assert_eq!("tuple", tokens.next().unwrap().to_string());
775 assert_eq!(".", tokens.next().unwrap().to_string());
776 assert_eq!("0.0", tokens.next().unwrap().to_string());
777 assert!(tokens.next().is_none());
778 }
779
780 #[cfg(span_locations)]
781 #[test]
non_ascii_tokens()782 fn non_ascii_tokens() {
783 check_spans("// abc", &[]);
784 check_spans("// ábc", &[]);
785 check_spans("// abc x", &[]);
786 check_spans("// ábc x", &[]);
787 check_spans("/* abc */ x", &[(1, 10, 1, 11)]);
788 check_spans("/* ábc */ x", &[(1, 10, 1, 11)]);
789 check_spans("/* ab\nc */ x", &[(2, 5, 2, 6)]);
790 check_spans("/* áb\nc */ x", &[(2, 5, 2, 6)]);
791 check_spans("/*** abc */ x", &[(1, 12, 1, 13)]);
792 check_spans("/*** ábc */ x", &[(1, 12, 1, 13)]);
793 check_spans(r#""abc""#, &[(1, 0, 1, 5)]);
794 check_spans(r#""ábc""#, &[(1, 0, 1, 5)]);
795 check_spans(r##"r#"abc"#"##, &[(1, 0, 1, 8)]);
796 check_spans(r##"r#"ábc"#"##, &[(1, 0, 1, 8)]);
797 check_spans("r#\"a\nc\"#", &[(1, 0, 2, 3)]);
798 check_spans("r#\"á\nc\"#", &[(1, 0, 2, 3)]);
799 check_spans("'a'", &[(1, 0, 1, 3)]);
800 check_spans("'á'", &[(1, 0, 1, 3)]);
801 check_spans("//! abc", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
802 check_spans("//! ábc", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
803 check_spans("//! abc\n", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
804 check_spans("//! ábc\n", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
805 check_spans("/*! abc */", &[(1, 0, 1, 10), (1, 0, 1, 10), (1, 0, 1, 10)]);
806 check_spans("/*! ábc */", &[(1, 0, 1, 10), (1, 0, 1, 10), (1, 0, 1, 10)]);
807 check_spans("/*! a\nc */", &[(1, 0, 2, 4), (1, 0, 2, 4), (1, 0, 2, 4)]);
808 check_spans("/*! á\nc */", &[(1, 0, 2, 4), (1, 0, 2, 4), (1, 0, 2, 4)]);
809 check_spans("abc", &[(1, 0, 1, 3)]);
810 check_spans("ábc", &[(1, 0, 1, 3)]);
811 check_spans("ábć", &[(1, 0, 1, 3)]);
812 check_spans("abc// foo", &[(1, 0, 1, 3)]);
813 check_spans("ábc// foo", &[(1, 0, 1, 3)]);
814 check_spans("ábć// foo", &[(1, 0, 1, 3)]);
815 check_spans("b\"a\\\n c\"", &[(1, 0, 2, 3)]);
816 }
817
818 #[cfg(span_locations)]
check_spans(p: &str, mut lines: &[(usize, usize, usize, usize)])819 fn check_spans(p: &str, mut lines: &[(usize, usize, usize, usize)]) {
820 let ts = p.parse::<TokenStream>().unwrap();
821 check_spans_internal(ts, &mut lines);
822 assert!(lines.is_empty(), "leftover ranges: {:?}", lines);
823 }
824
825 #[cfg(span_locations)]
check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usize)])826 fn check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usize)]) {
827 for i in ts {
828 if let Some((&(sline, scol, eline, ecol), rest)) = lines.split_first() {
829 *lines = rest;
830
831 let start = i.span().start();
832 assert_eq!(start.line, sline, "sline did not match for {}", i);
833 assert_eq!(start.column, scol, "scol did not match for {}", i);
834
835 let end = i.span().end();
836 assert_eq!(end.line, eline, "eline did not match for {}", i);
837 assert_eq!(end.column, ecol, "ecol did not match for {}", i);
838
839 if let TokenTree::Group(g) = i {
840 check_spans_internal(g.stream().clone(), lines);
841 }
842 }
843 }
844 }
845
846 #[test]
whitespace()847 fn whitespace() {
848 // space, horizontal tab, vertical tab, form feed, carriage return, line
849 // feed, non-breaking space, left-to-right mark, right-to-left mark
850 let various_spaces = " \t\u{b}\u{c}\r\n\u{a0}\u{200e}\u{200f}";
851 let tokens = various_spaces.parse::<TokenStream>().unwrap();
852 assert_eq!(tokens.into_iter().count(), 0);
853
854 let lone_carriage_returns = " \r \r\r\n ";
855 lone_carriage_returns.parse::<TokenStream>().unwrap();
856 }
857
858 #[test]
byte_order_mark()859 fn byte_order_mark() {
860 let string = "\u{feff}foo";
861 let tokens = string.parse::<TokenStream>().unwrap();
862 match tokens.into_iter().next().unwrap() {
863 TokenTree::Ident(ident) => assert_eq!(ident, "foo"),
864 _ => unreachable!(),
865 }
866
867 let string = "foo\u{feff}";
868 string.parse::<TokenStream>().unwrap_err();
869 }
870
871 #[cfg(span_locations)]
create_span() -> proc_macro2::Span872 fn create_span() -> proc_macro2::Span {
873 let tts: TokenStream = "1".parse().unwrap();
874 match tts.into_iter().next().unwrap() {
875 TokenTree::Literal(literal) => literal.span(),
876 _ => unreachable!(),
877 }
878 }
879
880 #[cfg(span_locations)]
881 #[test]
test_invalidate_current_thread_spans()882 fn test_invalidate_current_thread_spans() {
883 let actual = format!("{:#?}", create_span());
884 assert_eq!(actual, "bytes(1..2)");
885 let actual = format!("{:#?}", create_span());
886 assert_eq!(actual, "bytes(3..4)");
887
888 proc_macro2::extra::invalidate_current_thread_spans();
889
890 let actual = format!("{:#?}", create_span());
891 // Test that span offsets have been reset after the call
892 // to invalidate_current_thread_spans()
893 assert_eq!(actual, "bytes(1..2)");
894 }
895
896 #[cfg(span_locations)]
897 #[test]
898 #[should_panic(expected = "Invalid span with no related FileInfo!")]
test_use_span_after_invalidation()899 fn test_use_span_after_invalidation() {
900 let span = create_span();
901
902 proc_macro2::extra::invalidate_current_thread_spans();
903
904 span.source_text();
905 }
906