1 //! A "Parser" structure for token trees. We use this when parsing a declarative 2 //! macro definition into a list of patterns and templates. 3 4 use smallvec::{smallvec, SmallVec}; 5 use syntax::SyntaxKind; 6 7 use crate::{to_parser_input::to_parser_input, tt, ExpandError, ExpandResult}; 8 9 #[derive(Debug, Clone)] 10 pub(crate) struct TtIter<'a> { 11 pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>, 12 } 13 14 impl<'a> TtIter<'a> { new(subtree: &'a tt::Subtree) -> TtIter<'a>15 pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> { 16 TtIter { inner: subtree.token_trees.iter() } 17 } 18 expect_char(&mut self, char: char) -> Result<(), ()>19 pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> { 20 match self.next() { 21 Some(&tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if c == char => { 22 Ok(()) 23 } 24 _ => Err(()), 25 } 26 } 27 expect_any_char(&mut self, chars: &[char]) -> Result<(), ()>28 pub(crate) fn expect_any_char(&mut self, chars: &[char]) -> Result<(), ()> { 29 match self.next() { 30 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) 31 if chars.contains(c) => 32 { 33 Ok(()) 34 } 35 _ => Err(()), 36 } 37 } 38 expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()>39 pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> { 40 match self.next() { 41 Some(tt::TokenTree::Subtree(it)) => Ok(it), 42 _ => Err(()), 43 } 44 } 45 expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()>46 pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> { 47 match self.next() { 48 Some(tt::TokenTree::Leaf(it)) => Ok(it), 49 _ => Err(()), 50 } 51 } 52 expect_ident(&mut self) -> Result<&'a tt::Ident, ()>53 pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> { 54 match self.expect_leaf()? { 55 tt::Leaf::Ident(it) if it.text != "_" => Ok(it), 56 _ => Err(()), 57 } 58 } 59 expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident, ()>60 pub(crate) fn expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident, ()> { 61 match self.expect_leaf()? { 62 tt::Leaf::Ident(it) => Ok(it), 63 _ => Err(()), 64 } 65 } 66 expect_literal(&mut self) -> Result<&'a tt::Leaf, ()>67 pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> { 68 let it = self.expect_leaf()?; 69 match it { 70 tt::Leaf::Literal(_) => Ok(it), 71 tt::Leaf::Ident(ident) if ident.text == "true" || ident.text == "false" => Ok(it), 72 _ => Err(()), 73 } 74 } 75 expect_single_punct(&mut self) -> Result<&'a tt::Punct, ()>76 pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct, ()> { 77 match self.expect_leaf()? { 78 tt::Leaf::Punct(it) => Ok(it), 79 _ => Err(()), 80 } 81 } 82 83 /// Returns consecutive `Punct`s that can be glued together. 84 /// 85 /// This method currently may return a single quotation, which is part of lifetime ident and 86 /// conceptually not a punct in the context of mbe. Callers should handle this. expect_glued_punct(&mut self) -> Result<SmallVec<[tt::Punct; 3]>, ()>87 pub(crate) fn expect_glued_punct(&mut self) -> Result<SmallVec<[tt::Punct; 3]>, ()> { 88 let tt::TokenTree::Leaf(tt::Leaf::Punct(first)) = self.next().ok_or(())?.clone() else { 89 return Err(()); 90 }; 91 92 if first.spacing == tt::Spacing::Alone { 93 return Ok(smallvec![first]); 94 } 95 96 let (second, third) = match (self.peek_n(0), self.peek_n(1)) { 97 ( 98 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), 99 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))), 100 ) if p2.spacing == tt::Spacing::Joint => (p2, Some(p3)), 101 (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2, None), 102 _ => return Ok(smallvec![first]), 103 }; 104 105 match (first.char, second.char, third.map(|it| it.char)) { 106 ('.', '.', Some('.' | '=')) | ('<', '<', Some('=')) | ('>', '>', Some('=')) => { 107 let _ = self.next().unwrap(); 108 let _ = self.next().unwrap(); 109 Ok(smallvec![first, *second, *third.unwrap()]) 110 } 111 ('-' | '!' | '*' | '/' | '&' | '%' | '^' | '+' | '<' | '=' | '>' | '|', '=', _) 112 | ('-' | '=' | '>', '>', _) 113 | ('<', '-', _) 114 | (':', ':', _) 115 | ('.', '.', _) 116 | ('&', '&', _) 117 | ('<', '<', _) 118 | ('|', '|', _) => { 119 let _ = self.next().unwrap(); 120 Ok(smallvec![first, *second]) 121 } 122 _ => Ok(smallvec![first]), 123 } 124 } 125 expect_fragment( &mut self, entry_point: parser::PrefixEntryPoint, ) -> ExpandResult<Option<tt::TokenTree>>126 pub(crate) fn expect_fragment( 127 &mut self, 128 entry_point: parser::PrefixEntryPoint, 129 ) -> ExpandResult<Option<tt::TokenTree>> { 130 let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice()); 131 let parser_input = to_parser_input(&buffer); 132 let tree_traversal = entry_point.parse(&parser_input); 133 134 let mut cursor = buffer.begin(); 135 let mut error = false; 136 for step in tree_traversal.iter() { 137 match step { 138 parser::Step::Token { kind, mut n_input_tokens } => { 139 if kind == SyntaxKind::LIFETIME_IDENT { 140 n_input_tokens = 2; 141 } 142 for _ in 0..n_input_tokens { 143 cursor = cursor.bump_subtree(); 144 } 145 } 146 parser::Step::FloatSplit { .. } => { 147 // FIXME: We need to split the tree properly here, but mutating the token trees 148 // in the buffer is somewhat tricky to pull off. 149 cursor = cursor.bump_subtree(); 150 } 151 parser::Step::Enter { .. } | parser::Step::Exit => (), 152 parser::Step::Error { .. } => error = true, 153 } 154 } 155 156 let err = if error || !cursor.is_root() { 157 Some(ExpandError::binding_error(format!("expected {entry_point:?}"))) 158 } else { 159 None 160 }; 161 162 let mut curr = buffer.begin(); 163 let mut res = vec![]; 164 165 if cursor.is_root() { 166 while curr != cursor { 167 let Some(token) = curr.token_tree() else { break }; 168 res.push(token.cloned()); 169 curr = curr.bump(); 170 } 171 } 172 173 self.inner = self.inner.as_slice()[res.len()..].iter(); 174 let res = match res.len() { 175 0 | 1 => res.pop(), 176 _ => Some(tt::TokenTree::Subtree(tt::Subtree { 177 delimiter: tt::Delimiter::unspecified(), 178 token_trees: res, 179 })), 180 }; 181 ExpandResult { value: res, err } 182 } 183 peek_n(&self, n: usize) -> Option<&'a tt::TokenTree>184 pub(crate) fn peek_n(&self, n: usize) -> Option<&'a tt::TokenTree> { 185 self.inner.as_slice().get(n) 186 } 187 } 188 189 impl<'a> Iterator for TtIter<'a> { 190 type Item = &'a tt::TokenTree; next(&mut self) -> Option<Self::Item>191 fn next(&mut self) -> Option<Self::Item> { 192 self.inner.next() 193 } 194 size_hint(&self) -> (usize, Option<usize>)195 fn size_hint(&self) -> (usize, Option<usize>) { 196 self.inner.size_hint() 197 } 198 } 199 200 impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {} 201