1 //! Tokens representing Rust punctuation, keywords, and delimiters.
2 //!
3 //! The type names in this module can be difficult to keep straight, so we
4 //! prefer to use the [`Token!`] macro instead. This is a type-macro that
5 //! expands to the token type of the given token.
6 //!
7 //! [`Token!`]: crate::Token
8 //!
9 //! # Example
10 //!
11 //! The [`ItemStatic`] syntax tree node is defined like this.
12 //!
13 //! [`ItemStatic`]: crate::ItemStatic
14 //!
15 //! ```
16 //! # use syn::{Attribute, Expr, Ident, Token, Type, Visibility};
17 //! #
18 //! pub struct ItemStatic {
19 //! pub attrs: Vec<Attribute>,
20 //! pub vis: Visibility,
21 //! pub static_token: Token![static],
22 //! pub mutability: Option<Token![mut]>,
23 //! pub ident: Ident,
24 //! pub colon_token: Token![:],
25 //! pub ty: Box<Type>,
26 //! pub eq_token: Token![=],
27 //! pub expr: Box<Expr>,
28 //! pub semi_token: Token![;],
29 //! }
30 //! ```
31 //!
32 //! # Parsing
33 //!
34 //! Keywords and punctuation can be parsed through the [`ParseStream::parse`]
35 //! method. Delimiter tokens are parsed using the [`parenthesized!`],
36 //! [`bracketed!`] and [`braced!`] macros.
37 //!
38 //! [`ParseStream::parse`]: crate::parse::ParseBuffer::parse()
39 //! [`parenthesized!`]: crate::parenthesized!
40 //! [`bracketed!`]: crate::bracketed!
41 //! [`braced!`]: crate::braced!
42 //!
43 //! ```
44 //! use syn::{Attribute, Result};
45 //! use syn::parse::{Parse, ParseStream};
46 //! #
47 //! # enum ItemStatic {}
48 //!
49 //! // Parse the ItemStatic struct shown above.
50 //! impl Parse for ItemStatic {
51 //! fn parse(input: ParseStream) -> Result<Self> {
52 //! # use syn::ItemStatic;
53 //! # fn parse(input: ParseStream) -> Result<ItemStatic> {
54 //! Ok(ItemStatic {
55 //! attrs: input.call(Attribute::parse_outer)?,
56 //! vis: input.parse()?,
57 //! static_token: input.parse()?,
58 //! mutability: input.parse()?,
59 //! ident: input.parse()?,
60 //! colon_token: input.parse()?,
61 //! ty: input.parse()?,
62 //! eq_token: input.parse()?,
63 //! expr: input.parse()?,
64 //! semi_token: input.parse()?,
65 //! })
66 //! # }
67 //! # unimplemented!()
68 //! }
69 //! }
70 //! ```
71 //!
72 //! # Other operations
73 //!
74 //! Every keyword and punctuation token supports the following operations.
75 //!
76 //! - [Peeking] — `input.peek(Token![...])`
77 //!
78 //! - [Parsing] — `input.parse::<Token![...]>()?`
79 //!
80 //! - [Printing] — `quote!( ... #the_token ... )`
81 //!
82 //! - Construction from a [`Span`] — `let the_token = Token`
83 //!
84 //! - Field access to its span — `let sp = the_token.span`
85 //!
86 //! [Peeking]: crate::parse::ParseBuffer::peek()
87 //! [Parsing]: crate::parse::ParseBuffer::parse()
88 //! [Printing]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html
89 //! [`Span`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html
90
91 #[cfg(feature = "parsing")]
92 pub(crate) use self::private::CustomToken;
93 use self::private::WithSpan;
94 #[cfg(feature = "parsing")]
95 use crate::buffer::Cursor;
96 #[cfg(feature = "parsing")]
97 use crate::error::Result;
98 #[cfg(feature = "parsing")]
99 use crate::lifetime::Lifetime;
100 #[cfg(feature = "parsing")]
101 use crate::lit::{Lit, LitBool, LitByte, LitByteStr, LitChar, LitFloat, LitInt, LitStr};
102 #[cfg(feature = "parsing")]
103 use crate::lookahead;
104 #[cfg(feature = "parsing")]
105 use crate::parse::{Parse, ParseStream};
106 use crate::span::IntoSpans;
107 use proc_macro2::extra::DelimSpan;
108 use proc_macro2::Span;
109 #[cfg(feature = "printing")]
110 use proc_macro2::TokenStream;
111 #[cfg(any(feature = "parsing", feature = "printing"))]
112 use proc_macro2::{Delimiter, Ident};
113 #[cfg(feature = "parsing")]
114 use proc_macro2::{Literal, Punct, TokenTree};
115 #[cfg(feature = "printing")]
116 use quote::{ToTokens, TokenStreamExt};
117 #[cfg(feature = "extra-traits")]
118 use std::cmp;
119 #[cfg(feature = "extra-traits")]
120 use std::fmt::{self, Debug};
121 #[cfg(feature = "extra-traits")]
122 use std::hash::{Hash, Hasher};
123 use std::ops::{Deref, DerefMut};
124
125 /// Marker trait for types that represent single tokens.
126 ///
127 /// This trait is sealed and cannot be implemented for types outside of Syn.
128 #[cfg(feature = "parsing")]
129 pub trait Token: private::Sealed {
130 // Not public API.
131 #[doc(hidden)]
peek(cursor: Cursor) -> bool132 fn peek(cursor: Cursor) -> bool;
133
134 // Not public API.
135 #[doc(hidden)]
display() -> &'static str136 fn display() -> &'static str;
137 }
138
139 pub(crate) mod private {
140 #[cfg(feature = "parsing")]
141 use crate::buffer::Cursor;
142 use proc_macro2::Span;
143
144 #[cfg(feature = "parsing")]
145 pub trait Sealed {}
146
147 /// Support writing `token.span` rather than `token.spans[0]` on tokens that
148 /// hold a single span.
149 #[repr(transparent)]
150 #[allow(unknown_lints, repr_transparent_external_private_fields)] // False positive: https://github.com/rust-lang/rust/issues/78586#issuecomment-1722680482
151 pub struct WithSpan {
152 pub span: Span,
153 }
154
155 // Not public API.
156 #[doc(hidden)]
157 #[cfg(feature = "parsing")]
158 pub trait CustomToken {
peek(cursor: Cursor) -> bool159 fn peek(cursor: Cursor) -> bool;
display() -> &'static str160 fn display() -> &'static str;
161 }
162 }
163
164 #[cfg(feature = "parsing")]
165 impl private::Sealed for Ident {}
166
167 #[cfg(feature = "parsing")]
peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool168 fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool {
169 use crate::parse::Unexpected;
170 use std::cell::Cell;
171 use std::rc::Rc;
172
173 let scope = Span::call_site();
174 let unexpected = Rc::new(Cell::new(Unexpected::None));
175 let buffer = crate::parse::new_parse_buffer(scope, cursor, unexpected);
176 peek(&buffer)
177 }
178
179 macro_rules! impl_token {
180 ($display:literal $name:ty) => {
181 #[cfg(feature = "parsing")]
182 impl Token for $name {
183 fn peek(cursor: Cursor) -> bool {
184 fn peek(input: ParseStream) -> bool {
185 <$name as Parse>::parse(input).is_ok()
186 }
187 peek_impl(cursor, peek)
188 }
189
190 fn display() -> &'static str {
191 $display
192 }
193 }
194
195 #[cfg(feature = "parsing")]
196 impl private::Sealed for $name {}
197 };
198 }
199
200 impl_token!("lifetime" Lifetime);
201 impl_token!("literal" Lit);
202 impl_token!("string literal" LitStr);
203 impl_token!("byte string literal" LitByteStr);
204 impl_token!("byte literal" LitByte);
205 impl_token!("character literal" LitChar);
206 impl_token!("integer literal" LitInt);
207 impl_token!("floating point literal" LitFloat);
208 impl_token!("boolean literal" LitBool);
209 impl_token!("group token" proc_macro2::Group);
210
211 macro_rules! impl_low_level_token {
212 ($display:literal $ty:ident $get:ident) => {
213 #[cfg(feature = "parsing")]
214 impl Token for $ty {
215 fn peek(cursor: Cursor) -> bool {
216 cursor.$get().is_some()
217 }
218
219 fn display() -> &'static str {
220 $display
221 }
222 }
223
224 #[cfg(feature = "parsing")]
225 impl private::Sealed for $ty {}
226 };
227 }
228
229 impl_low_level_token!("punctuation token" Punct punct);
230 impl_low_level_token!("literal" Literal literal);
231 impl_low_level_token!("token" TokenTree token_tree);
232
233 #[cfg(feature = "parsing")]
234 impl<T: CustomToken> private::Sealed for T {}
235
236 #[cfg(feature = "parsing")]
237 impl<T: CustomToken> Token for T {
peek(cursor: Cursor) -> bool238 fn peek(cursor: Cursor) -> bool {
239 <Self as CustomToken>::peek(cursor)
240 }
241
display() -> &'static str242 fn display() -> &'static str {
243 <Self as CustomToken>::display()
244 }
245 }
246
247 macro_rules! define_keywords {
248 ($($token:literal pub struct $name:ident)*) => {
249 $(
250 #[doc = concat!('`', $token, '`')]
251 ///
252 /// Don't try to remember the name of this type — use the
253 /// [`Token!`] macro instead.
254 ///
255 /// [`Token!`]: crate::token
256 pub struct $name {
257 pub span: Span,
258 }
259
260 #[doc(hidden)]
261 #[allow(non_snake_case)]
262 pub fn $name<S: IntoSpans<Span>>(span: S) -> $name {
263 $name {
264 span: span.into_spans(),
265 }
266 }
267
268 impl std::default::Default for $name {
269 fn default() -> Self {
270 $name {
271 span: Span::call_site(),
272 }
273 }
274 }
275
276 #[cfg(feature = "clone-impls")]
277 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
278 impl Copy for $name {}
279
280 #[cfg(feature = "clone-impls")]
281 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
282 impl Clone for $name {
283 fn clone(&self) -> Self {
284 *self
285 }
286 }
287
288 #[cfg(feature = "extra-traits")]
289 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
290 impl Debug for $name {
291 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
292 f.write_str(stringify!($name))
293 }
294 }
295
296 #[cfg(feature = "extra-traits")]
297 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
298 impl cmp::Eq for $name {}
299
300 #[cfg(feature = "extra-traits")]
301 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
302 impl PartialEq for $name {
303 fn eq(&self, _other: &$name) -> bool {
304 true
305 }
306 }
307
308 #[cfg(feature = "extra-traits")]
309 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
310 impl Hash for $name {
311 fn hash<H: Hasher>(&self, _state: &mut H) {}
312 }
313
314 #[cfg(feature = "printing")]
315 #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))]
316 impl ToTokens for $name {
317 fn to_tokens(&self, tokens: &mut TokenStream) {
318 printing::keyword($token, self.span, tokens);
319 }
320 }
321
322 #[cfg(feature = "parsing")]
323 #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
324 impl Parse for $name {
325 fn parse(input: ParseStream) -> Result<Self> {
326 Ok($name {
327 span: parsing::keyword(input, $token)?,
328 })
329 }
330 }
331
332 #[cfg(feature = "parsing")]
333 impl Token for $name {
334 fn peek(cursor: Cursor) -> bool {
335 parsing::peek_keyword(cursor, $token)
336 }
337
338 fn display() -> &'static str {
339 concat!("`", $token, "`")
340 }
341 }
342
343 #[cfg(feature = "parsing")]
344 impl private::Sealed for $name {}
345 )*
346 };
347 }
348
349 macro_rules! impl_deref_if_len_is_1 {
350 ($name:ident/1) => {
351 impl Deref for $name {
352 type Target = WithSpan;
353
354 fn deref(&self) -> &Self::Target {
355 unsafe { &*(self as *const Self).cast::<WithSpan>() }
356 }
357 }
358
359 impl DerefMut for $name {
360 fn deref_mut(&mut self) -> &mut Self::Target {
361 unsafe { &mut *(self as *mut Self).cast::<WithSpan>() }
362 }
363 }
364 };
365
366 ($name:ident/$len:literal) => {};
367 }
368
369 macro_rules! define_punctuation_structs {
370 ($($token:literal pub struct $name:ident/$len:tt #[doc = $usage:literal])*) => {
371 $(
372 #[cfg_attr(not(doc), repr(transparent))]
373 #[allow(unknown_lints, repr_transparent_external_private_fields)] // False positive: https://github.com/rust-lang/rust/issues/78586#issuecomment-1722680482
374 #[doc = concat!('`', $token, '`')]
375 ///
376 /// Usage:
377 #[doc = concat!($usage, '.')]
378 ///
379 /// Don't try to remember the name of this type — use the
380 /// [`Token!`] macro instead.
381 ///
382 /// [`Token!`]: crate::token
383 pub struct $name {
384 pub spans: [Span; $len],
385 }
386
387 #[doc(hidden)]
388 #[allow(non_snake_case)]
389 pub fn $name<S: IntoSpans<[Span; $len]>>(spans: S) -> $name {
390 $name {
391 spans: spans.into_spans(),
392 }
393 }
394
395 impl std::default::Default for $name {
396 fn default() -> Self {
397 $name {
398 spans: [Span::call_site(); $len],
399 }
400 }
401 }
402
403 #[cfg(feature = "clone-impls")]
404 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
405 impl Copy for $name {}
406
407 #[cfg(feature = "clone-impls")]
408 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
409 impl Clone for $name {
410 fn clone(&self) -> Self {
411 *self
412 }
413 }
414
415 #[cfg(feature = "extra-traits")]
416 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
417 impl Debug for $name {
418 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
419 f.write_str(stringify!($name))
420 }
421 }
422
423 #[cfg(feature = "extra-traits")]
424 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
425 impl cmp::Eq for $name {}
426
427 #[cfg(feature = "extra-traits")]
428 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
429 impl PartialEq for $name {
430 fn eq(&self, _other: &$name) -> bool {
431 true
432 }
433 }
434
435 #[cfg(feature = "extra-traits")]
436 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
437 impl Hash for $name {
438 fn hash<H: Hasher>(&self, _state: &mut H) {}
439 }
440
441 impl_deref_if_len_is_1!($name/$len);
442 )*
443 };
444 }
445
446 macro_rules! define_punctuation {
447 ($($token:literal pub struct $name:ident/$len:tt #[doc = $usage:literal])*) => {
448 $(
449 define_punctuation_structs! {
450 $token pub struct $name/$len #[doc = $usage]
451 }
452
453 #[cfg(feature = "printing")]
454 #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))]
455 impl ToTokens for $name {
456 fn to_tokens(&self, tokens: &mut TokenStream) {
457 printing::punct($token, &self.spans, tokens);
458 }
459 }
460
461 #[cfg(feature = "parsing")]
462 #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
463 impl Parse for $name {
464 fn parse(input: ParseStream) -> Result<Self> {
465 Ok($name {
466 spans: parsing::punct(input, $token)?,
467 })
468 }
469 }
470
471 #[cfg(feature = "parsing")]
472 impl Token for $name {
473 fn peek(cursor: Cursor) -> bool {
474 parsing::peek_punct(cursor, $token)
475 }
476
477 fn display() -> &'static str {
478 concat!("`", $token, "`")
479 }
480 }
481
482 #[cfg(feature = "parsing")]
483 impl private::Sealed for $name {}
484 )*
485 };
486 }
487
488 macro_rules! define_delimiters {
489 ($($delim:ident pub struct $name:ident #[$doc:meta])*) => {
490 $(
491 #[$doc]
492 pub struct $name {
493 pub span: DelimSpan,
494 }
495
496 #[doc(hidden)]
497 #[allow(non_snake_case)]
498 pub fn $name<S: IntoSpans<DelimSpan>>(span: S) -> $name {
499 $name {
500 span: span.into_spans(),
501 }
502 }
503
504 impl std::default::Default for $name {
505 fn default() -> Self {
506 $name(Span::call_site())
507 }
508 }
509
510 #[cfg(feature = "clone-impls")]
511 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
512 impl Copy for $name {}
513
514 #[cfg(feature = "clone-impls")]
515 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
516 impl Clone for $name {
517 fn clone(&self) -> Self {
518 *self
519 }
520 }
521
522 #[cfg(feature = "extra-traits")]
523 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
524 impl Debug for $name {
525 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
526 f.write_str(stringify!($name))
527 }
528 }
529
530 #[cfg(feature = "extra-traits")]
531 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
532 impl cmp::Eq for $name {}
533
534 #[cfg(feature = "extra-traits")]
535 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
536 impl PartialEq for $name {
537 fn eq(&self, _other: &$name) -> bool {
538 true
539 }
540 }
541
542 #[cfg(feature = "extra-traits")]
543 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
544 impl Hash for $name {
545 fn hash<H: Hasher>(&self, _state: &mut H) {}
546 }
547
548 impl $name {
549 #[cfg(feature = "printing")]
550 pub fn surround<F>(&self, tokens: &mut TokenStream, f: F)
551 where
552 F: FnOnce(&mut TokenStream),
553 {
554 let mut inner = TokenStream::new();
555 f(&mut inner);
556 printing::delim(Delimiter::$delim, self.span.join(), tokens, inner);
557 }
558 }
559
560 #[cfg(feature = "parsing")]
561 impl private::Sealed for $name {}
562 )*
563 };
564 }
565
566 define_punctuation_structs! {
567 "_" pub struct Underscore/1 /// wildcard patterns, inferred types, unnamed items in constants, extern crates, use declarations, and destructuring assignment
568 }
569
570 #[cfg(feature = "printing")]
571 #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))]
572 impl ToTokens for Underscore {
to_tokens(&self, tokens: &mut TokenStream)573 fn to_tokens(&self, tokens: &mut TokenStream) {
574 tokens.append(Ident::new("_", self.span));
575 }
576 }
577
578 #[cfg(feature = "parsing")]
579 #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
580 impl Parse for Underscore {
parse(input: ParseStream) -> Result<Self>581 fn parse(input: ParseStream) -> Result<Self> {
582 input.step(|cursor| {
583 if let Some((ident, rest)) = cursor.ident() {
584 if ident == "_" {
585 return Ok((Underscore(ident.span()), rest));
586 }
587 }
588 if let Some((punct, rest)) = cursor.punct() {
589 if punct.as_char() == '_' {
590 return Ok((Underscore(punct.span()), rest));
591 }
592 }
593 Err(cursor.error("expected `_`"))
594 })
595 }
596 }
597
598 #[cfg(feature = "parsing")]
599 impl Token for Underscore {
peek(cursor: Cursor) -> bool600 fn peek(cursor: Cursor) -> bool {
601 if let Some((ident, _rest)) = cursor.ident() {
602 return ident == "_";
603 }
604 if let Some((punct, _rest)) = cursor.punct() {
605 return punct.as_char() == '_';
606 }
607 false
608 }
609
display() -> &'static str610 fn display() -> &'static str {
611 "`_`"
612 }
613 }
614
615 #[cfg(feature = "parsing")]
616 impl private::Sealed for Underscore {}
617
618 /// None-delimited group
619 pub struct Group {
620 pub span: Span,
621 }
622
623 #[doc(hidden)]
624 #[allow(non_snake_case)]
Group<S: IntoSpans<Span>>(span: S) -> Group625 pub fn Group<S: IntoSpans<Span>>(span: S) -> Group {
626 Group {
627 span: span.into_spans(),
628 }
629 }
630
631 impl std::default::Default for Group {
default() -> Self632 fn default() -> Self {
633 Group {
634 span: Span::call_site(),
635 }
636 }
637 }
638
639 #[cfg(feature = "clone-impls")]
640 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
641 impl Copy for Group {}
642
643 #[cfg(feature = "clone-impls")]
644 #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))]
645 impl Clone for Group {
clone(&self) -> Self646 fn clone(&self) -> Self {
647 *self
648 }
649 }
650
651 #[cfg(feature = "extra-traits")]
652 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
653 impl Debug for Group {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result654 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
655 f.write_str("Group")
656 }
657 }
658
659 #[cfg(feature = "extra-traits")]
660 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
661 impl cmp::Eq for Group {}
662
663 #[cfg(feature = "extra-traits")]
664 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
665 impl PartialEq for Group {
eq(&self, _other: &Group) -> bool666 fn eq(&self, _other: &Group) -> bool {
667 true
668 }
669 }
670
671 #[cfg(feature = "extra-traits")]
672 #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
673 impl Hash for Group {
hash<H: Hasher>(&self, _state: &mut H)674 fn hash<H: Hasher>(&self, _state: &mut H) {}
675 }
676
677 impl Group {
678 #[cfg(feature = "printing")]
surround<F>(&self, tokens: &mut TokenStream, f: F) where F: FnOnce(&mut TokenStream),679 pub fn surround<F>(&self, tokens: &mut TokenStream, f: F)
680 where
681 F: FnOnce(&mut TokenStream),
682 {
683 let mut inner = TokenStream::new();
684 f(&mut inner);
685 printing::delim(Delimiter::None, self.span, tokens, inner);
686 }
687 }
688
689 #[cfg(feature = "parsing")]
690 impl private::Sealed for Group {}
691
692 #[cfg(feature = "parsing")]
693 impl Token for Paren {
peek(cursor: Cursor) -> bool694 fn peek(cursor: Cursor) -> bool {
695 lookahead::is_delimiter(cursor, Delimiter::Parenthesis)
696 }
697
display() -> &'static str698 fn display() -> &'static str {
699 "parentheses"
700 }
701 }
702
703 #[cfg(feature = "parsing")]
704 impl Token for Brace {
peek(cursor: Cursor) -> bool705 fn peek(cursor: Cursor) -> bool {
706 lookahead::is_delimiter(cursor, Delimiter::Brace)
707 }
708
display() -> &'static str709 fn display() -> &'static str {
710 "curly braces"
711 }
712 }
713
714 #[cfg(feature = "parsing")]
715 impl Token for Bracket {
peek(cursor: Cursor) -> bool716 fn peek(cursor: Cursor) -> bool {
717 lookahead::is_delimiter(cursor, Delimiter::Bracket)
718 }
719
display() -> &'static str720 fn display() -> &'static str {
721 "square brackets"
722 }
723 }
724
725 #[cfg(feature = "parsing")]
726 impl Token for Group {
peek(cursor: Cursor) -> bool727 fn peek(cursor: Cursor) -> bool {
728 lookahead::is_delimiter(cursor, Delimiter::None)
729 }
730
display() -> &'static str731 fn display() -> &'static str {
732 "invisible group"
733 }
734 }
735
736 define_keywords! {
737 "abstract" pub struct Abstract
738 "as" pub struct As
739 "async" pub struct Async
740 "auto" pub struct Auto
741 "await" pub struct Await
742 "become" pub struct Become
743 "box" pub struct Box
744 "break" pub struct Break
745 "const" pub struct Const
746 "continue" pub struct Continue
747 "crate" pub struct Crate
748 "default" pub struct Default
749 "do" pub struct Do
750 "dyn" pub struct Dyn
751 "else" pub struct Else
752 "enum" pub struct Enum
753 "extern" pub struct Extern
754 "final" pub struct Final
755 "fn" pub struct Fn
756 "for" pub struct For
757 "if" pub struct If
758 "impl" pub struct Impl
759 "in" pub struct In
760 "let" pub struct Let
761 "loop" pub struct Loop
762 "macro" pub struct Macro
763 "match" pub struct Match
764 "mod" pub struct Mod
765 "move" pub struct Move
766 "mut" pub struct Mut
767 "override" pub struct Override
768 "priv" pub struct Priv
769 "pub" pub struct Pub
770 "ref" pub struct Ref
771 "return" pub struct Return
772 "Self" pub struct SelfType
773 "self" pub struct SelfValue
774 "static" pub struct Static
775 "struct" pub struct Struct
776 "super" pub struct Super
777 "trait" pub struct Trait
778 "try" pub struct Try
779 "type" pub struct Type
780 "typeof" pub struct Typeof
781 "union" pub struct Union
782 "unsafe" pub struct Unsafe
783 "unsized" pub struct Unsized
784 "use" pub struct Use
785 "virtual" pub struct Virtual
786 "where" pub struct Where
787 "while" pub struct While
788 "yield" pub struct Yield
789 }
790
791 define_punctuation! {
792 "&" pub struct And/1 /// bitwise and logical AND, borrow, references, reference patterns
793 "&&" pub struct AndAnd/2 /// lazy AND, borrow, references, reference patterns
794 "&=" pub struct AndEq/2 /// bitwise AND assignment
795 "@" pub struct At/1 /// subpattern binding
796 "^" pub struct Caret/1 /// bitwise and logical XOR
797 "^=" pub struct CaretEq/2 /// bitwise XOR assignment
798 ":" pub struct Colon/1 /// various separators
799 "," pub struct Comma/1 /// various separators
800 "$" pub struct Dollar/1 /// macros
801 "." pub struct Dot/1 /// field access, tuple index
802 ".." pub struct DotDot/2 /// range, struct expressions, patterns, range patterns
803 "..." pub struct DotDotDot/3 /// variadic functions, range patterns
804 "..=" pub struct DotDotEq/3 /// inclusive range, range patterns
805 "=" pub struct Eq/1 /// assignment, attributes, various type definitions
806 "==" pub struct EqEq/2 /// equal
807 "=>" pub struct FatArrow/2 /// match arms, macros
808 ">=" pub struct Ge/2 /// greater than or equal to, generics
809 ">" pub struct Gt/1 /// greater than, generics, paths
810 "<-" pub struct LArrow/2 /// unused
811 "<=" pub struct Le/2 /// less than or equal to
812 "<" pub struct Lt/1 /// less than, generics, paths
813 "-" pub struct Minus/1 /// subtraction, negation
814 "-=" pub struct MinusEq/2 /// subtraction assignment
815 "!=" pub struct Ne/2 /// not equal
816 "!" pub struct Not/1 /// bitwise and logical NOT, macro calls, inner attributes, never type, negative impls
817 "|" pub struct Or/1 /// bitwise and logical OR, closures, patterns in match, if let, and while let
818 "|=" pub struct OrEq/2 /// bitwise OR assignment
819 "||" pub struct OrOr/2 /// lazy OR, closures
820 "::" pub struct PathSep/2 /// path separator
821 "%" pub struct Percent/1 /// remainder
822 "%=" pub struct PercentEq/2 /// remainder assignment
823 "+" pub struct Plus/1 /// addition, trait bounds, macro Kleene matcher
824 "+=" pub struct PlusEq/2 /// addition assignment
825 "#" pub struct Pound/1 /// attributes
826 "?" pub struct Question/1 /// question mark operator, questionably sized, macro Kleene matcher
827 "->" pub struct RArrow/2 /// function return type, closure return type, function pointer type
828 ";" pub struct Semi/1 /// terminator for various items and statements, array types
829 "<<" pub struct Shl/2 /// shift left, nested generics
830 "<<=" pub struct ShlEq/3 /// shift left assignment
831 ">>" pub struct Shr/2 /// shift right, nested generics
832 ">>=" pub struct ShrEq/3 /// shift right assignment, nested generics
833 "/" pub struct Slash/1 /// division
834 "/=" pub struct SlashEq/2 /// division assignment
835 "*" pub struct Star/1 /// multiplication, dereference, raw pointers, macro Kleene matcher, use wildcards
836 "*=" pub struct StarEq/2 /// multiplication assignment
837 "~" pub struct Tilde/1 /// unused since before Rust 1.0
838 }
839
840 define_delimiters! {
841 Brace pub struct Brace /// `{`…`}`
842 Bracket pub struct Bracket /// `[`…`]`
843 Parenthesis pub struct Paren /// `(`…`)`
844 }
845
846 /// A type-macro that expands to the name of the Rust type representation of a
847 /// given token.
848 ///
849 /// As a type, `Token!` is commonly used in the type of struct fields, the type
850 /// of a `let` statement, or in turbofish for a `parse` function.
851 ///
852 /// ```
853 /// use syn::{Ident, Token};
854 /// use syn::parse::{Parse, ParseStream, Result};
855 ///
856 /// // `struct Foo;`
857 /// pub struct UnitStruct {
858 /// struct_token: Token![struct],
859 /// ident: Ident,
860 /// semi_token: Token![;],
861 /// }
862 ///
863 /// impl Parse for UnitStruct {
864 /// fn parse(input: ParseStream) -> Result<Self> {
865 /// let struct_token: Token![struct] = input.parse()?;
866 /// let ident: Ident = input.parse()?;
867 /// let semi_token = input.parse::<Token![;]>()?;
868 /// Ok(UnitStruct { struct_token, ident, semi_token })
869 /// }
870 /// }
871 /// ```
872 ///
873 /// As an expression, `Token!` is used for peeking tokens or instantiating
874 /// tokens from a span.
875 ///
876 /// ```
877 /// # use syn::{Ident, Token};
878 /// # use syn::parse::{Parse, ParseStream, Result};
879 /// #
880 /// # struct UnitStruct {
881 /// # struct_token: Token![struct],
882 /// # ident: Ident,
883 /// # semi_token: Token![;],
884 /// # }
885 /// #
886 /// # impl Parse for UnitStruct {
887 /// # fn parse(input: ParseStream) -> Result<Self> {
888 /// # unimplemented!()
889 /// # }
890 /// # }
891 /// #
892 /// fn make_unit_struct(name: Ident) -> UnitStruct {
893 /// let span = name.span();
894 /// UnitStruct {
895 /// struct_token: Token,
896 /// ident: name,
897 /// semi_token: Token,
898 /// }
899 /// }
900 ///
901 /// # fn parse(input: ParseStream) -> Result<()> {
902 /// if input.peek(Token![struct]) {
903 /// let unit_struct: UnitStruct = input.parse()?;
904 /// /* ... */
905 /// }
906 /// # Ok(())
907 /// # }
908 /// ```
909 ///
910 /// See the [token module] documentation for details and examples.
911 ///
912 /// [token module]: crate::token
913 #[macro_export]
914 macro_rules! Token {
915 [abstract] => { $crate::token::Abstract };
916 [as] => { $crate::token::As };
917 [async] => { $crate::token::Async };
918 [auto] => { $crate::token::Auto };
919 [await] => { $crate::token::Await };
920 [become] => { $crate::token::Become };
921 [box] => { $crate::token::Box };
922 [break] => { $crate::token::Break };
923 [const] => { $crate::token::Const };
924 [continue] => { $crate::token::Continue };
925 [crate] => { $crate::token::Crate };
926 [default] => { $crate::token::Default };
927 [do] => { $crate::token::Do };
928 [dyn] => { $crate::token::Dyn };
929 [else] => { $crate::token::Else };
930 [enum] => { $crate::token::Enum };
931 [extern] => { $crate::token::Extern };
932 [final] => { $crate::token::Final };
933 [fn] => { $crate::token::Fn };
934 [for] => { $crate::token::For };
935 [if] => { $crate::token::If };
936 [impl] => { $crate::token::Impl };
937 [in] => { $crate::token::In };
938 [let] => { $crate::token::Let };
939 [loop] => { $crate::token::Loop };
940 [macro] => { $crate::token::Macro };
941 [match] => { $crate::token::Match };
942 [mod] => { $crate::token::Mod };
943 [move] => { $crate::token::Move };
944 [mut] => { $crate::token::Mut };
945 [override] => { $crate::token::Override };
946 [priv] => { $crate::token::Priv };
947 [pub] => { $crate::token::Pub };
948 [ref] => { $crate::token::Ref };
949 [return] => { $crate::token::Return };
950 [Self] => { $crate::token::SelfType };
951 [self] => { $crate::token::SelfValue };
952 [static] => { $crate::token::Static };
953 [struct] => { $crate::token::Struct };
954 [super] => { $crate::token::Super };
955 [trait] => { $crate::token::Trait };
956 [try] => { $crate::token::Try };
957 [type] => { $crate::token::Type };
958 [typeof] => { $crate::token::Typeof };
959 [union] => { $crate::token::Union };
960 [unsafe] => { $crate::token::Unsafe };
961 [unsized] => { $crate::token::Unsized };
962 [use] => { $crate::token::Use };
963 [virtual] => { $crate::token::Virtual };
964 [where] => { $crate::token::Where };
965 [while] => { $crate::token::While };
966 [yield] => { $crate::token::Yield };
967 [&] => { $crate::token::And };
968 [&&] => { $crate::token::AndAnd };
969 [&=] => { $crate::token::AndEq };
970 [@] => { $crate::token::At };
971 [^] => { $crate::token::Caret };
972 [^=] => { $crate::token::CaretEq };
973 [:] => { $crate::token::Colon };
974 [,] => { $crate::token::Comma };
975 [$] => { $crate::token::Dollar };
976 [.] => { $crate::token::Dot };
977 [..] => { $crate::token::DotDot };
978 [...] => { $crate::token::DotDotDot };
979 [..=] => { $crate::token::DotDotEq };
980 [=] => { $crate::token::Eq };
981 [==] => { $crate::token::EqEq };
982 [=>] => { $crate::token::FatArrow };
983 [>=] => { $crate::token::Ge };
984 [>] => { $crate::token::Gt };
985 [<-] => { $crate::token::LArrow };
986 [<=] => { $crate::token::Le };
987 [<] => { $crate::token::Lt };
988 [-] => { $crate::token::Minus };
989 [-=] => { $crate::token::MinusEq };
990 [!=] => { $crate::token::Ne };
991 [!] => { $crate::token::Not };
992 [|] => { $crate::token::Or };
993 [|=] => { $crate::token::OrEq };
994 [||] => { $crate::token::OrOr };
995 [::] => { $crate::token::PathSep };
996 [%] => { $crate::token::Percent };
997 [%=] => { $crate::token::PercentEq };
998 [+] => { $crate::token::Plus };
999 [+=] => { $crate::token::PlusEq };
1000 [#] => { $crate::token::Pound };
1001 [?] => { $crate::token::Question };
1002 [->] => { $crate::token::RArrow };
1003 [;] => { $crate::token::Semi };
1004 [<<] => { $crate::token::Shl };
1005 [<<=] => { $crate::token::ShlEq };
1006 [>>] => { $crate::token::Shr };
1007 [>>=] => { $crate::token::ShrEq };
1008 [/] => { $crate::token::Slash };
1009 [/=] => { $crate::token::SlashEq };
1010 [*] => { $crate::token::Star };
1011 [*=] => { $crate::token::StarEq };
1012 [~] => { $crate::token::Tilde };
1013 [_] => { $crate::token::Underscore };
1014 }
1015
1016 // Not public API.
1017 #[doc(hidden)]
1018 #[cfg(feature = "parsing")]
1019 pub(crate) mod parsing {
1020 use crate::buffer::Cursor;
1021 use crate::error::{Error, Result};
1022 use crate::parse::ParseStream;
1023 use proc_macro2::{Spacing, Span};
1024
keyword(input: ParseStream, token: &str) -> Result<Span>1025 pub(crate) fn keyword(input: ParseStream, token: &str) -> Result<Span> {
1026 input.step(|cursor| {
1027 if let Some((ident, rest)) = cursor.ident() {
1028 if ident == token {
1029 return Ok((ident.span(), rest));
1030 }
1031 }
1032 Err(cursor.error(format!("expected `{}`", token)))
1033 })
1034 }
1035
peek_keyword(cursor: Cursor, token: &str) -> bool1036 pub(crate) fn peek_keyword(cursor: Cursor, token: &str) -> bool {
1037 if let Some((ident, _rest)) = cursor.ident() {
1038 ident == token
1039 } else {
1040 false
1041 }
1042 }
1043
1044 #[doc(hidden)]
punct<const N: usize>(input: ParseStream, token: &str) -> Result<[Span; N]>1045 pub fn punct<const N: usize>(input: ParseStream, token: &str) -> Result<[Span; N]> {
1046 let mut spans = [input.span(); N];
1047 punct_helper(input, token, &mut spans)?;
1048 Ok(spans)
1049 }
1050
punct_helper(input: ParseStream, token: &str, spans: &mut [Span]) -> Result<()>1051 fn punct_helper(input: ParseStream, token: &str, spans: &mut [Span]) -> Result<()> {
1052 input.step(|cursor| {
1053 let mut cursor = *cursor;
1054 assert_eq!(token.len(), spans.len());
1055
1056 for (i, ch) in token.chars().enumerate() {
1057 match cursor.punct() {
1058 Some((punct, rest)) => {
1059 spans[i] = punct.span();
1060 if punct.as_char() != ch {
1061 break;
1062 } else if i == token.len() - 1 {
1063 return Ok(((), rest));
1064 } else if punct.spacing() != Spacing::Joint {
1065 break;
1066 }
1067 cursor = rest;
1068 }
1069 None => break,
1070 }
1071 }
1072
1073 Err(Error::new(spans[0], format!("expected `{}`", token)))
1074 })
1075 }
1076
1077 #[doc(hidden)]
peek_punct(mut cursor: Cursor, token: &str) -> bool1078 pub fn peek_punct(mut cursor: Cursor, token: &str) -> bool {
1079 for (i, ch) in token.chars().enumerate() {
1080 match cursor.punct() {
1081 Some((punct, rest)) => {
1082 if punct.as_char() != ch {
1083 break;
1084 } else if i == token.len() - 1 {
1085 return true;
1086 } else if punct.spacing() != Spacing::Joint {
1087 break;
1088 }
1089 cursor = rest;
1090 }
1091 None => break,
1092 }
1093 }
1094 false
1095 }
1096 }
1097
1098 // Not public API.
1099 #[doc(hidden)]
1100 #[cfg(feature = "printing")]
1101 pub(crate) mod printing {
1102 use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream};
1103 use quote::TokenStreamExt;
1104
1105 #[doc(hidden)]
punct(s: &str, spans: &[Span], tokens: &mut TokenStream)1106 pub fn punct(s: &str, spans: &[Span], tokens: &mut TokenStream) {
1107 assert_eq!(s.len(), spans.len());
1108
1109 let mut chars = s.chars();
1110 let mut spans = spans.iter();
1111 let ch = chars.next_back().unwrap();
1112 let span = spans.next_back().unwrap();
1113 for (ch, span) in chars.zip(spans) {
1114 let mut op = Punct::new(ch, Spacing::Joint);
1115 op.set_span(*span);
1116 tokens.append(op);
1117 }
1118
1119 let mut op = Punct::new(ch, Spacing::Alone);
1120 op.set_span(*span);
1121 tokens.append(op);
1122 }
1123
keyword(s: &str, span: Span, tokens: &mut TokenStream)1124 pub(crate) fn keyword(s: &str, span: Span, tokens: &mut TokenStream) {
1125 tokens.append(Ident::new(s, span));
1126 }
1127
delim( delim: Delimiter, span: Span, tokens: &mut TokenStream, inner: TokenStream, )1128 pub(crate) fn delim(
1129 delim: Delimiter,
1130 span: Span,
1131 tokens: &mut TokenStream,
1132 inner: TokenStream,
1133 ) {
1134 let mut g = Group::new(delim, inner);
1135 g.set_span(span);
1136 tokens.append(g);
1137 }
1138 }
1139