1 //! Tokens representing Rust punctuation, keywords, and delimiters.
2 //!
3 //! The type names in this module can be difficult to keep straight, so we
4 //! prefer to use the [`Token!`] macro instead. This is a type-macro that
5 //! expands to the token type of the given token.
6 //!
7 //! [`Token!`]: crate::Token
8 //!
9 //! # Example
10 //!
11 //! The [`ItemStatic`] syntax tree node is defined like this.
12 //!
13 //! [`ItemStatic`]: crate::ItemStatic
14 //!
15 //! ```
16 //! # use syn::{Attribute, Expr, Ident, Token, Type, Visibility};
17 //! #
18 //! pub struct ItemStatic {
19 //! pub attrs: Vec<Attribute>,
20 //! pub vis: Visibility,
21 //! pub static_token: Token![static],
22 //! pub mutability: Option<Token![mut]>,
23 //! pub ident: Ident,
24 //! pub colon_token: Token![:],
25 //! pub ty: Box<Type>,
26 //! pub eq_token: Token![=],
27 //! pub expr: Box<Expr>,
28 //! pub semi_token: Token![;],
29 //! }
30 //! ```
31 //!
32 //! # Parsing
33 //!
34 //! Keywords and punctuation can be parsed through the [`ParseStream::parse`]
35 //! method. Delimiter tokens are parsed using the [`parenthesized!`],
36 //! [`bracketed!`] and [`braced!`] macros.
37 //!
38 //! [`ParseStream::parse`]: crate::parse::ParseBuffer::parse()
39 //! [`parenthesized!`]: crate::parenthesized!
40 //! [`bracketed!`]: crate::bracketed!
41 //! [`braced!`]: crate::braced!
42 //!
43 //! ```
44 //! use syn::{Attribute, Result};
45 //! use syn::parse::{Parse, ParseStream};
46 //! #
47 //! # enum ItemStatic {}
48 //!
49 //! // Parse the ItemStatic struct shown above.
50 //! impl Parse for ItemStatic {
51 //! fn parse(input: ParseStream) -> Result<Self> {
52 //! # use syn::ItemStatic;
53 //! # fn parse(input: ParseStream) -> Result<ItemStatic> {
54 //! Ok(ItemStatic {
55 //! attrs: input.call(Attribute::parse_outer)?,
56 //! vis: input.parse()?,
57 //! static_token: input.parse()?,
58 //! mutability: input.parse()?,
59 //! ident: input.parse()?,
60 //! colon_token: input.parse()?,
61 //! ty: input.parse()?,
62 //! eq_token: input.parse()?,
63 //! expr: input.parse()?,
64 //! semi_token: input.parse()?,
65 //! })
66 //! # }
67 //! # unimplemented!()
68 //! }
69 //! }
70 //! ```
71 //!
72 //! # Other operations
73 //!
74 //! Every keyword and punctuation token supports the following operations.
75 //!
76 //! - [Peeking] — `input.peek(Token![...])`
77 //!
78 //! - [Parsing] — `input.parse::<Token![...]>()?`
79 //!
80 //! - [Printing] — `quote!( ... #the_token ... )`
81 //!
82 //! - Construction from a [`Span`] — `let the_token = Token`
83 //!
84 //! - Field access to its span — `let sp = the_token.span`
85 //!
86 //! [Peeking]: crate::parse::ParseBuffer::peek()
87 //! [Parsing]: crate::parse::ParseBuffer::parse()
88 //! [Printing]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html
89 //! [`Span`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html
90
91 use self::private::WithSpan;
92 #[cfg(feature = "parsing")]
93 use crate::buffer::Cursor;
94 #[cfg(feature = "parsing")]
95 use crate::error::Result;
96 #[cfg(feature = "parsing")]
97 use crate::lifetime::Lifetime;
98 #[cfg(feature = "parsing")]
99 use crate::lit::{Lit, LitBool, LitByte, LitByteStr, LitChar, LitFloat, LitInt, LitStr};
100 #[cfg(feature = "parsing")]
101 use crate::lookahead;
102 #[cfg(feature = "parsing")]
103 use crate::parse::{Parse, ParseStream};
104 use crate::span::IntoSpans;
105 #[cfg(any(feature = "parsing", feature = "printing"))]
106 use proc_macro2::Ident;
107 use proc_macro2::Span;
108 #[cfg(feature = "printing")]
109 use proc_macro2::TokenStream;
110 #[cfg(feature = "parsing")]
111 use proc_macro2::{Delimiter, Literal, Punct, TokenTree};
112 #[cfg(feature = "printing")]
113 use quote::{ToTokens, TokenStreamExt};
114 #[cfg(feature = "extra-traits")]
115 use std::cmp;
116 #[cfg(feature = "extra-traits")]
117 use std::fmt::{self, Debug};
118 #[cfg(feature = "extra-traits")]
119 use std::hash::{Hash, Hasher};
120 use std::ops::{Deref, DerefMut};
121
122 /// Marker trait for types that represent single tokens.
123 ///
124 /// This trait is sealed and cannot be implemented for types outside of Syn.
125 #[cfg(feature = "parsing")]
126 pub trait Token: private::Sealed {
127 // Not public API.
128 #[doc(hidden)]
peek(cursor: Cursor) -> bool129 fn peek(cursor: Cursor) -> bool;
130
131 // Not public API.
132 #[doc(hidden)]
display() -> &'static str133 fn display() -> &'static str;
134 }
135
136 mod private {
137 use proc_macro2::Span;
138
139 #[cfg(feature = "parsing")]
140 pub trait Sealed {}
141
142 /// Support writing `token.span` rather than `token.spans[0]` on tokens that
143 /// hold a single span.
144 #[repr(C)]
145 pub struct WithSpan {
146 pub span: Span,
147 }
148 }
149
150 #[cfg(feature = "parsing")]
151 impl private::Sealed for Ident {}
152
153 #[cfg(feature = "parsing")]
peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool154 fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool {
155 use crate::parse::Unexpected;
156 use std::cell::Cell;
157 use std::rc::Rc;
158
159 let scope = Span::call_site();
160 let unexpected = Rc::new(Cell::new(Unexpected::None));
161 let buffer = crate::parse::new_parse_buffer(scope, cursor, unexpected);
162 peek(&buffer)
163 }
164
165 macro_rules! impl_token {
166 ($display:tt $name:ty) => {
167 #[cfg(feature = "parsing")]
168 impl Token for $name {
169 fn peek(cursor: Cursor) -> bool {
170 fn peek(input: ParseStream) -> bool {
171 <$name as Parse>::parse(input).is_ok()
172 }
173 peek_impl(cursor, peek)
174 }
175
176 fn display() -> &'static str {
177 $display
178 }
179 }
180
181 #[cfg(feature = "parsing")]
182 impl private::Sealed for $name {}
183 };
184 }
185
186 impl_token!("lifetime" Lifetime);
187 impl_token!("literal" Lit);
188 impl_token!("string literal" LitStr);
189 impl_token!("byte string literal" LitByteStr);
190 impl_token!("byte literal" LitByte);
191 impl_token!("character literal" LitChar);
192 impl_token!("integer literal" LitInt);
193 impl_token!("floating point literal" LitFloat);
194 impl_token!("boolean literal" LitBool);
195 impl_token!("group token" proc_macro2::Group);
196
197 macro_rules! impl_low_level_token {
198 ($display:tt $ty:ident $get:ident) => {
199 #[cfg(feature = "parsing")]
200 impl Token for $ty {
201 fn peek(cursor: Cursor) -> bool {
202 cursor.$get().is_some()
203 }
204
205 fn display() -> &'static str {
206 $display
207 }
208 }
209
210 #[cfg(feature = "parsing")]
211 impl private::Sealed for $ty {}
212 };
213 }
214
215 impl_low_level_token!("punctuation token" Punct punct);
216 impl_low_level_token!("literal" Literal literal);
217 impl_low_level_token!("token" TokenTree token_tree);
218
219 // Not public API.
220 #[doc(hidden)]
221 #[cfg(feature = "parsing")]
222 pub trait CustomToken {
peek(cursor: Cursor) -> bool223 fn peek(cursor: Cursor) -> bool;
display() -> &'static str224 fn display() -> &'static str;
225 }
226
227 #[cfg(feature = "parsing")]
228 impl<T: CustomToken> private::Sealed for T {}
229
230 #[cfg(feature = "parsing")]
231 impl<T: CustomToken> Token for T {
peek(cursor: Cursor) -> bool232 fn peek(cursor: Cursor) -> bool {
233 <Self as CustomToken>::peek(cursor)
234 }
235
display() -> &'static str236 fn display() -> &'static str {
237 <Self as CustomToken>::display()
238 }
239 }
240
241 macro_rules! define_keywords {
242 ($($token:tt pub struct $name:ident #[$doc:meta])*) => {
243 $(
244 #[$doc]
245 ///
246 /// Don't try to remember the name of this type — use the
247 /// [`Token!`] macro instead.
248 ///
249 /// [`Token!`]: crate::token
250 pub struct $name {
251 pub span: Span,
252 }
253
254 #[doc(hidden)]
255 #[allow(non_snake_case)]
256 pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name {
257 $name {
258 span: span.into_spans()[0],
259 }
260 }
261
262 impl std::default::Default for $name {
263 fn default() -> Self {
264 $name {
265 span: Span::call_site(),
266 }
267 }
268 }
269
270 #[cfg(feature = "clone-impls")]
271 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
272 impl Copy for $name {}
273
274 #[cfg(feature = "clone-impls")]
275 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
276 impl Clone for $name {
277 fn clone(&self) -> Self {
278 *self
279 }
280 }
281
282 #[cfg(feature = "extra-traits")]
283 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
284 impl Debug for $name {
285 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
286 f.write_str(stringify!($name))
287 }
288 }
289
290 #[cfg(feature = "extra-traits")]
291 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
292 impl cmp::Eq for $name {}
293
294 #[cfg(feature = "extra-traits")]
295 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
296 impl PartialEq for $name {
297 fn eq(&self, _other: &$name) -> bool {
298 true
299 }
300 }
301
302 #[cfg(feature = "extra-traits")]
303 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
304 impl Hash for $name {
305 fn hash<H: Hasher>(&self, _state: &mut H) {}
306 }
307
308 #[cfg(feature = "printing")]
309 #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))]
310 impl ToTokens for $name {
311 fn to_tokens(&self, tokens: &mut TokenStream) {
312 printing::keyword($token, self.span, tokens);
313 }
314 }
315
316 #[cfg(feature = "parsing")]
317 #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
318 impl Parse for $name {
319 fn parse(input: ParseStream) -> Result<Self> {
320 Ok($name {
321 span: parsing::keyword(input, $token)?,
322 })
323 }
324 }
325
326 #[cfg(feature = "parsing")]
327 impl Token for $name {
328 fn peek(cursor: Cursor) -> bool {
329 parsing::peek_keyword(cursor, $token)
330 }
331
332 fn display() -> &'static str {
333 concat!("`", $token, "`")
334 }
335 }
336
337 #[cfg(feature = "parsing")]
338 impl private::Sealed for $name {}
339 )*
340 };
341 }
342
343 macro_rules! impl_deref_if_len_is_1 {
344 ($name:ident/1) => {
345 impl Deref for $name {
346 type Target = WithSpan;
347
348 fn deref(&self) -> &Self::Target {
349 unsafe { &*(self as *const Self as *const WithSpan) }
350 }
351 }
352
353 impl DerefMut for $name {
354 fn deref_mut(&mut self) -> &mut Self::Target {
355 unsafe { &mut *(self as *mut Self as *mut WithSpan) }
356 }
357 }
358 };
359
360 ($name:ident/$len:tt) => {};
361 }
362
363 macro_rules! define_punctuation_structs {
364 ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => {
365 $(
366 #[repr(C)]
367 #[$doc]
368 ///
369 /// Don't try to remember the name of this type — use the
370 /// [`Token!`] macro instead.
371 ///
372 /// [`Token!`]: crate::token
373 pub struct $name {
374 pub spans: [Span; $len],
375 }
376
377 #[doc(hidden)]
378 #[allow(non_snake_case)]
379 pub fn $name<S: IntoSpans<[Span; $len]>>(spans: S) -> $name {
380 $name {
381 spans: spans.into_spans(),
382 }
383 }
384
385 impl std::default::Default for $name {
386 fn default() -> Self {
387 $name {
388 spans: [Span::call_site(); $len],
389 }
390 }
391 }
392
393 #[cfg(feature = "clone-impls")]
394 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
395 impl Copy for $name {}
396
397 #[cfg(feature = "clone-impls")]
398 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
399 impl Clone for $name {
400 fn clone(&self) -> Self {
401 *self
402 }
403 }
404
405 #[cfg(feature = "extra-traits")]
406 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
407 impl Debug for $name {
408 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
409 f.write_str(stringify!($name))
410 }
411 }
412
413 #[cfg(feature = "extra-traits")]
414 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
415 impl cmp::Eq for $name {}
416
417 #[cfg(feature = "extra-traits")]
418 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
419 impl PartialEq for $name {
420 fn eq(&self, _other: &$name) -> bool {
421 true
422 }
423 }
424
425 #[cfg(feature = "extra-traits")]
426 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
427 impl Hash for $name {
428 fn hash<H: Hasher>(&self, _state: &mut H) {}
429 }
430
431 impl_deref_if_len_is_1!($name/$len);
432 )*
433 };
434 }
435
436 macro_rules! define_punctuation {
437 ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => {
438 $(
439 define_punctuation_structs! {
440 $token pub struct $name/$len #[$doc]
441 }
442
443 #[cfg(feature = "printing")]
444 #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))]
445 impl ToTokens for $name {
446 fn to_tokens(&self, tokens: &mut TokenStream) {
447 printing::punct($token, &self.spans, tokens);
448 }
449 }
450
451 #[cfg(feature = "parsing")]
452 #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
453 impl Parse for $name {
454 fn parse(input: ParseStream) -> Result<Self> {
455 Ok($name {
456 spans: parsing::punct(input, $token)?,
457 })
458 }
459 }
460
461 #[cfg(feature = "parsing")]
462 impl Token for $name {
463 fn peek(cursor: Cursor) -> bool {
464 parsing::peek_punct(cursor, $token)
465 }
466
467 fn display() -> &'static str {
468 concat!("`", $token, "`")
469 }
470 }
471
472 #[cfg(feature = "parsing")]
473 impl private::Sealed for $name {}
474 )*
475 };
476 }
477
478 macro_rules! define_delimiters {
479 ($($token:tt pub struct $name:ident #[$doc:meta])*) => {
480 $(
481 #[$doc]
482 pub struct $name {
483 pub span: Span,
484 }
485
486 #[doc(hidden)]
487 #[allow(non_snake_case)]
488 pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name {
489 $name {
490 span: span.into_spans()[0],
491 }
492 }
493
494 impl std::default::Default for $name {
495 fn default() -> Self {
496 $name {
497 span: Span::call_site(),
498 }
499 }
500 }
501
502 #[cfg(feature = "clone-impls")]
503 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
504 impl Copy for $name {}
505
506 #[cfg(feature = "clone-impls")]
507 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
508 impl Clone for $name {
509 fn clone(&self) -> Self {
510 *self
511 }
512 }
513
514 #[cfg(feature = "extra-traits")]
515 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
516 impl Debug for $name {
517 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
518 f.write_str(stringify!($name))
519 }
520 }
521
522 #[cfg(feature = "extra-traits")]
523 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
524 impl cmp::Eq for $name {}
525
526 #[cfg(feature = "extra-traits")]
527 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
528 impl PartialEq for $name {
529 fn eq(&self, _other: &$name) -> bool {
530 true
531 }
532 }
533
534 #[cfg(feature = "extra-traits")]
535 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
536 impl Hash for $name {
537 fn hash<H: Hasher>(&self, _state: &mut H) {}
538 }
539
540 impl $name {
541 #[cfg(feature = "printing")]
542 pub fn surround<F>(&self, tokens: &mut TokenStream, f: F)
543 where
544 F: FnOnce(&mut TokenStream),
545 {
546 printing::delim($token, self.span, tokens, f);
547 }
548 }
549
550 #[cfg(feature = "parsing")]
551 impl private::Sealed for $name {}
552 )*
553 };
554 }
555
556 define_punctuation_structs! {
557 "_" pub struct Underscore/1 /// `_`
558 }
559
560 #[cfg(feature = "printing")]
561 #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))]
562 impl ToTokens for Underscore {
to_tokens(&self, tokens: &mut TokenStream)563 fn to_tokens(&self, tokens: &mut TokenStream) {
564 tokens.append(Ident::new("_", self.span));
565 }
566 }
567
568 #[cfg(feature = "parsing")]
569 #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
570 impl Parse for Underscore {
parse(input: ParseStream) -> Result<Self>571 fn parse(input: ParseStream) -> Result<Self> {
572 input.step(|cursor| {
573 if let Some((ident, rest)) = cursor.ident() {
574 if ident == "_" {
575 return Ok((Underscore(ident.span()), rest));
576 }
577 }
578 if let Some((punct, rest)) = cursor.punct() {
579 if punct.as_char() == '_' {
580 return Ok((Underscore(punct.span()), rest));
581 }
582 }
583 Err(cursor.error("expected `_`"))
584 })
585 }
586 }
587
588 #[cfg(feature = "parsing")]
589 impl Token for Underscore {
peek(cursor: Cursor) -> bool590 fn peek(cursor: Cursor) -> bool {
591 if let Some((ident, _rest)) = cursor.ident() {
592 return ident == "_";
593 }
594 if let Some((punct, _rest)) = cursor.punct() {
595 return punct.as_char() == '_';
596 }
597 false
598 }
599
display() -> &'static str600 fn display() -> &'static str {
601 "`_`"
602 }
603 }
604
605 #[cfg(feature = "parsing")]
606 impl private::Sealed for Underscore {}
607
608 #[cfg(feature = "parsing")]
609 impl Token for Paren {
peek(cursor: Cursor) -> bool610 fn peek(cursor: Cursor) -> bool {
611 lookahead::is_delimiter(cursor, Delimiter::Parenthesis)
612 }
613
display() -> &'static str614 fn display() -> &'static str {
615 "parentheses"
616 }
617 }
618
619 #[cfg(feature = "parsing")]
620 impl Token for Brace {
peek(cursor: Cursor) -> bool621 fn peek(cursor: Cursor) -> bool {
622 lookahead::is_delimiter(cursor, Delimiter::Brace)
623 }
624
display() -> &'static str625 fn display() -> &'static str {
626 "curly braces"
627 }
628 }
629
630 #[cfg(feature = "parsing")]
631 impl Token for Bracket {
peek(cursor: Cursor) -> bool632 fn peek(cursor: Cursor) -> bool {
633 lookahead::is_delimiter(cursor, Delimiter::Bracket)
634 }
635
display() -> &'static str636 fn display() -> &'static str {
637 "square brackets"
638 }
639 }
640
641 #[cfg(feature = "parsing")]
642 impl Token for Group {
peek(cursor: Cursor) -> bool643 fn peek(cursor: Cursor) -> bool {
644 lookahead::is_delimiter(cursor, Delimiter::None)
645 }
646
display() -> &'static str647 fn display() -> &'static str {
648 "invisible group"
649 }
650 }
651
652 define_keywords! {
653 "abstract" pub struct Abstract /// `abstract`
654 "as" pub struct As /// `as`
655 "async" pub struct Async /// `async`
656 "auto" pub struct Auto /// `auto`
657 "await" pub struct Await /// `await`
658 "become" pub struct Become /// `become`
659 "box" pub struct Box /// `box`
660 "break" pub struct Break /// `break`
661 "const" pub struct Const /// `const`
662 "continue" pub struct Continue /// `continue`
663 "crate" pub struct Crate /// `crate`
664 "default" pub struct Default /// `default`
665 "do" pub struct Do /// `do`
666 "dyn" pub struct Dyn /// `dyn`
667 "else" pub struct Else /// `else`
668 "enum" pub struct Enum /// `enum`
669 "extern" pub struct Extern /// `extern`
670 "final" pub struct Final /// `final`
671 "fn" pub struct Fn /// `fn`
672 "for" pub struct For /// `for`
673 "if" pub struct If /// `if`
674 "impl" pub struct Impl /// `impl`
675 "in" pub struct In /// `in`
676 "let" pub struct Let /// `let`
677 "loop" pub struct Loop /// `loop`
678 "macro" pub struct Macro /// `macro`
679 "match" pub struct Match /// `match`
680 "mod" pub struct Mod /// `mod`
681 "move" pub struct Move /// `move`
682 "mut" pub struct Mut /// `mut`
683 "override" pub struct Override /// `override`
684 "priv" pub struct Priv /// `priv`
685 "pub" pub struct Pub /// `pub`
686 "ref" pub struct Ref /// `ref`
687 "return" pub struct Return /// `return`
688 "Self" pub struct SelfType /// `Self`
689 "self" pub struct SelfValue /// `self`
690 "static" pub struct Static /// `static`
691 "struct" pub struct Struct /// `struct`
692 "super" pub struct Super /// `super`
693 "trait" pub struct Trait /// `trait`
694 "try" pub struct Try /// `try`
695 "type" pub struct Type /// `type`
696 "typeof" pub struct Typeof /// `typeof`
697 "union" pub struct Union /// `union`
698 "unsafe" pub struct Unsafe /// `unsafe`
699 "unsized" pub struct Unsized /// `unsized`
700 "use" pub struct Use /// `use`
701 "virtual" pub struct Virtual /// `virtual`
702 "where" pub struct Where /// `where`
703 "while" pub struct While /// `while`
704 "yield" pub struct Yield /// `yield`
705 }
706
707 define_punctuation! {
708 "+" pub struct Add/1 /// `+`
709 "+=" pub struct AddEq/2 /// `+=`
710 "&" pub struct And/1 /// `&`
711 "&&" pub struct AndAnd/2 /// `&&`
712 "&=" pub struct AndEq/2 /// `&=`
713 "@" pub struct At/1 /// `@`
714 "!" pub struct Bang/1 /// `!`
715 "^" pub struct Caret/1 /// `^`
716 "^=" pub struct CaretEq/2 /// `^=`
717 ":" pub struct Colon/1 /// `:`
718 "::" pub struct Colon2/2 /// `::`
719 "," pub struct Comma/1 /// `,`
720 "/" pub struct Div/1 /// `/`
721 "/=" pub struct DivEq/2 /// `/=`
722 "$" pub struct Dollar/1 /// `$`
723 "." pub struct Dot/1 /// `.`
724 ".." pub struct Dot2/2 /// `..`
725 "..." pub struct Dot3/3 /// `...`
726 "..=" pub struct DotDotEq/3 /// `..=`
727 "=" pub struct Eq/1 /// `=`
728 "==" pub struct EqEq/2 /// `==`
729 ">=" pub struct Ge/2 /// `>=`
730 ">" pub struct Gt/1 /// `>`
731 "<=" pub struct Le/2 /// `<=`
732 "<" pub struct Lt/1 /// `<`
733 "*=" pub struct MulEq/2 /// `*=`
734 "!=" pub struct Ne/2 /// `!=`
735 "|" pub struct Or/1 /// `|`
736 "|=" pub struct OrEq/2 /// `|=`
737 "||" pub struct OrOr/2 /// `||`
738 "#" pub struct Pound/1 /// `#`
739 "?" pub struct Question/1 /// `?`
740 "->" pub struct RArrow/2 /// `->`
741 "<-" pub struct LArrow/2 /// `<-`
742 "%" pub struct Rem/1 /// `%`
743 "%=" pub struct RemEq/2 /// `%=`
744 "=>" pub struct FatArrow/2 /// `=>`
745 ";" pub struct Semi/1 /// `;`
746 "<<" pub struct Shl/2 /// `<<`
747 "<<=" pub struct ShlEq/3 /// `<<=`
748 ">>" pub struct Shr/2 /// `>>`
749 ">>=" pub struct ShrEq/3 /// `>>=`
750 "*" pub struct Star/1 /// `*`
751 "-" pub struct Sub/1 /// `-`
752 "-=" pub struct SubEq/2 /// `-=`
753 "~" pub struct Tilde/1 /// `~`
754 }
755
756 define_delimiters! {
757 "{" pub struct Brace /// `{...}`
758 "[" pub struct Bracket /// `[...]`
759 "(" pub struct Paren /// `(...)`
760 " " pub struct Group /// None-delimited group
761 }
762
763 macro_rules! export_token_macro {
764 ($($await_rule:tt)*) => {
765 /// A type-macro that expands to the name of the Rust type representation of a
766 /// given token.
767 ///
768 /// See the [token module] documentation for details and examples.
769 ///
770 /// [token module]: crate::token
771 // Unfortunate duplication due to a rustdoc bug.
772 // https://github.com/rust-lang/rust/issues/45939
773 #[macro_export]
774 macro_rules! Token {
775 [abstract] => { $crate::token::Abstract };
776 [as] => { $crate::token::As };
777 [async] => { $crate::token::Async };
778 [auto] => { $crate::token::Auto };
779 $($await_rule => { $crate::token::Await };)*
780 [become] => { $crate::token::Become };
781 [box] => { $crate::token::Box };
782 [break] => { $crate::token::Break };
783 [const] => { $crate::token::Const };
784 [continue] => { $crate::token::Continue };
785 [crate] => { $crate::token::Crate };
786 [default] => { $crate::token::Default };
787 [do] => { $crate::token::Do };
788 [dyn] => { $crate::token::Dyn };
789 [else] => { $crate::token::Else };
790 [enum] => { $crate::token::Enum };
791 [extern] => { $crate::token::Extern };
792 [final] => { $crate::token::Final };
793 [fn] => { $crate::token::Fn };
794 [for] => { $crate::token::For };
795 [if] => { $crate::token::If };
796 [impl] => { $crate::token::Impl };
797 [in] => { $crate::token::In };
798 [let] => { $crate::token::Let };
799 [loop] => { $crate::token::Loop };
800 [macro] => { $crate::token::Macro };
801 [match] => { $crate::token::Match };
802 [mod] => { $crate::token::Mod };
803 [move] => { $crate::token::Move };
804 [mut] => { $crate::token::Mut };
805 [override] => { $crate::token::Override };
806 [priv] => { $crate::token::Priv };
807 [pub] => { $crate::token::Pub };
808 [ref] => { $crate::token::Ref };
809 [return] => { $crate::token::Return };
810 [Self] => { $crate::token::SelfType };
811 [self] => { $crate::token::SelfValue };
812 [static] => { $crate::token::Static };
813 [struct] => { $crate::token::Struct };
814 [super] => { $crate::token::Super };
815 [trait] => { $crate::token::Trait };
816 [try] => { $crate::token::Try };
817 [type] => { $crate::token::Type };
818 [typeof] => { $crate::token::Typeof };
819 [union] => { $crate::token::Union };
820 [unsafe] => { $crate::token::Unsafe };
821 [unsized] => { $crate::token::Unsized };
822 [use] => { $crate::token::Use };
823 [virtual] => { $crate::token::Virtual };
824 [where] => { $crate::token::Where };
825 [while] => { $crate::token::While };
826 [yield] => { $crate::token::Yield };
827 [+] => { $crate::token::Add };
828 [+=] => { $crate::token::AddEq };
829 [&] => { $crate::token::And };
830 [&&] => { $crate::token::AndAnd };
831 [&=] => { $crate::token::AndEq };
832 [@] => { $crate::token::At };
833 [!] => { $crate::token::Bang };
834 [^] => { $crate::token::Caret };
835 [^=] => { $crate::token::CaretEq };
836 [:] => { $crate::token::Colon };
837 [::] => { $crate::token::Colon2 };
838 [,] => { $crate::token::Comma };
839 [/] => { $crate::token::Div };
840 [/=] => { $crate::token::DivEq };
841 [$] => { $crate::token::Dollar };
842 [.] => { $crate::token::Dot };
843 [..] => { $crate::token::Dot2 };
844 [...] => { $crate::token::Dot3 };
845 [..=] => { $crate::token::DotDotEq };
846 [=] => { $crate::token::Eq };
847 [==] => { $crate::token::EqEq };
848 [>=] => { $crate::token::Ge };
849 [>] => { $crate::token::Gt };
850 [<=] => { $crate::token::Le };
851 [<] => { $crate::token::Lt };
852 [*=] => { $crate::token::MulEq };
853 [!=] => { $crate::token::Ne };
854 [|] => { $crate::token::Or };
855 [|=] => { $crate::token::OrEq };
856 [||] => { $crate::token::OrOr };
857 [#] => { $crate::token::Pound };
858 [?] => { $crate::token::Question };
859 [->] => { $crate::token::RArrow };
860 [<-] => { $crate::token::LArrow };
861 [%] => { $crate::token::Rem };
862 [%=] => { $crate::token::RemEq };
863 [=>] => { $crate::token::FatArrow };
864 [;] => { $crate::token::Semi };
865 [<<] => { $crate::token::Shl };
866 [<<=] => { $crate::token::ShlEq };
867 [>>] => { $crate::token::Shr };
868 [>>=] => { $crate::token::ShrEq };
869 [*] => { $crate::token::Star };
870 [-] => { $crate::token::Sub };
871 [-=] => { $crate::token::SubEq };
872 [~] => { $crate::token::Tilde };
873 [_] => { $crate::token::Underscore };
874 }
875 };
876 }
877
878 // Old rustc does not permit `await` appearing anywhere in the source file.
879 // https://github.com/rust-lang/rust/issues/57919
880 // We put the Token![await] rule in a place that is not lexed by old rustc.
881 #[cfg(not(syn_omit_await_from_token_macro))]
882 include!("await.rs"); // export_token_macro! {[await]}
883 #[cfg(syn_omit_await_from_token_macro)]
884 export_token_macro! {}
885
886 // Not public API.
887 #[doc(hidden)]
888 #[cfg(feature = "parsing")]
889 pub mod parsing {
890 use crate::buffer::Cursor;
891 use crate::error::{Error, Result};
892 use crate::parse::ParseStream;
893 use crate::span::FromSpans;
894 use proc_macro2::{Spacing, Span};
895
keyword(input: ParseStream, token: &str) -> Result<Span>896 pub fn keyword(input: ParseStream, token: &str) -> Result<Span> {
897 input.step(|cursor| {
898 if let Some((ident, rest)) = cursor.ident() {
899 if ident == token {
900 return Ok((ident.span(), rest));
901 }
902 }
903 Err(cursor.error(format!("expected `{}`", token)))
904 })
905 }
906
peek_keyword(cursor: Cursor, token: &str) -> bool907 pub fn peek_keyword(cursor: Cursor, token: &str) -> bool {
908 if let Some((ident, _rest)) = cursor.ident() {
909 ident == token
910 } else {
911 false
912 }
913 }
914
punct<S: FromSpans>(input: ParseStream, token: &str) -> Result<S>915 pub fn punct<S: FromSpans>(input: ParseStream, token: &str) -> Result<S> {
916 let mut spans = [input.span(); 3];
917 punct_helper(input, token, &mut spans)?;
918 Ok(S::from_spans(&spans))
919 }
920
punct_helper(input: ParseStream, token: &str, spans: &mut [Span; 3]) -> Result<()>921 fn punct_helper(input: ParseStream, token: &str, spans: &mut [Span; 3]) -> Result<()> {
922 input.step(|cursor| {
923 let mut cursor = *cursor;
924 assert!(token.len() <= spans.len());
925
926 for (i, ch) in token.chars().enumerate() {
927 match cursor.punct() {
928 Some((punct, rest)) => {
929 spans[i] = punct.span();
930 if punct.as_char() != ch {
931 break;
932 } else if i == token.len() - 1 {
933 return Ok(((), rest));
934 } else if punct.spacing() != Spacing::Joint {
935 break;
936 }
937 cursor = rest;
938 }
939 None => break,
940 }
941 }
942
943 Err(Error::new(spans[0], format!("expected `{}`", token)))
944 })
945 }
946
peek_punct(mut cursor: Cursor, token: &str) -> bool947 pub fn peek_punct(mut cursor: Cursor, token: &str) -> bool {
948 for (i, ch) in token.chars().enumerate() {
949 match cursor.punct() {
950 Some((punct, rest)) => {
951 if punct.as_char() != ch {
952 break;
953 } else if i == token.len() - 1 {
954 return true;
955 } else if punct.spacing() != Spacing::Joint {
956 break;
957 }
958 cursor = rest;
959 }
960 None => break,
961 }
962 }
963 false
964 }
965 }
966
967 // Not public API.
968 #[doc(hidden)]
969 #[cfg(feature = "printing")]
970 pub mod printing {
971 use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream};
972 use quote::TokenStreamExt;
973
punct(s: &str, spans: &[Span], tokens: &mut TokenStream)974 pub fn punct(s: &str, spans: &[Span], tokens: &mut TokenStream) {
975 assert_eq!(s.len(), spans.len());
976
977 let mut chars = s.chars();
978 let mut spans = spans.iter();
979 let ch = chars.next_back().unwrap();
980 let span = spans.next_back().unwrap();
981 for (ch, span) in chars.zip(spans) {
982 let mut op = Punct::new(ch, Spacing::Joint);
983 op.set_span(*span);
984 tokens.append(op);
985 }
986
987 let mut op = Punct::new(ch, Spacing::Alone);
988 op.set_span(*span);
989 tokens.append(op);
990 }
991
keyword(s: &str, span: Span, tokens: &mut TokenStream)992 pub fn keyword(s: &str, span: Span, tokens: &mut TokenStream) {
993 tokens.append(Ident::new(s, span));
994 }
995
delim<F>(s: &str, span: Span, tokens: &mut TokenStream, f: F) where F: FnOnce(&mut TokenStream),996 pub fn delim<F>(s: &str, span: Span, tokens: &mut TokenStream, f: F)
997 where
998 F: FnOnce(&mut TokenStream),
999 {
1000 let delim = match s {
1001 "(" => Delimiter::Parenthesis,
1002 "[" => Delimiter::Bracket,
1003 "{" => Delimiter::Brace,
1004 " " => Delimiter::None,
1005 _ => panic!("unknown delimiter: {}", s),
1006 };
1007 let mut inner = TokenStream::new();
1008 f(&mut inner);
1009 let mut g = Group::new(delim, inner);
1010 g.set_span(span);
1011 tokens.append(g);
1012 }
1013 }
1014