1 // pest. The Elegant Parser
2 // Copyright (c) 2018 Dragoș Tiselice
3 //
4 // Licensed under the Apache License, Version 2.0
5 // <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
6 // license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7 // option. All files in the project carrying such notice may not be copied,
8 // modified, or distributed except according to those terms.
9
10 use std::path::PathBuf;
11
12 use proc_macro2::TokenStream;
13 use quote::{ToTokens, TokenStreamExt};
14 use syn::{self, Generics, Ident};
15
16 use pest::unicode::unicode_property_names;
17 use pest_meta::ast::*;
18 use pest_meta::optimizer::*;
19
20 use crate::docs::DocComment;
21
generate( name: Ident, generics: &Generics, path: Option<PathBuf>, rules: Vec<OptimizedRule>, defaults: Vec<&str>, doc_comment: &DocComment, include_grammar: bool, ) -> TokenStream22 pub(crate) fn generate(
23 name: Ident,
24 generics: &Generics,
25 path: Option<PathBuf>,
26 rules: Vec<OptimizedRule>,
27 defaults: Vec<&str>,
28 doc_comment: &DocComment,
29 include_grammar: bool,
30 ) -> TokenStream {
31 let uses_eoi = defaults.iter().any(|name| *name == "EOI");
32
33 let builtins = generate_builtin_rules();
34 let include_fix = if include_grammar {
35 match path {
36 Some(ref path) => generate_include(&name, path.to_str().expect("non-Unicode path")),
37 None => quote!(),
38 }
39 } else {
40 quote!()
41 };
42 let rule_enum = generate_enum(&rules, doc_comment, uses_eoi);
43 let patterns = generate_patterns(&rules, uses_eoi);
44 let skip = generate_skip(&rules);
45
46 let mut rules: Vec<_> = rules.into_iter().map(generate_rule).collect();
47 rules.extend(builtins.into_iter().filter_map(|(builtin, tokens)| {
48 if defaults.contains(&builtin) {
49 Some(tokens)
50 } else {
51 None
52 }
53 }));
54
55 let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
56
57 let result = result_type();
58
59 let parser_impl = quote! {
60 #[allow(clippy::all)]
61 impl #impl_generics ::pest::Parser<Rule> for #name #ty_generics #where_clause {
62 fn parse<'i>(
63 rule: Rule,
64 input: &'i str
65 ) -> #result<
66 ::pest::iterators::Pairs<'i, Rule>,
67 ::pest::error::Error<Rule>
68 > {
69 mod rules {
70 #![allow(clippy::upper_case_acronyms)]
71 pub mod hidden {
72 use super::super::Rule;
73 #skip
74 }
75
76 pub mod visible {
77 use super::super::Rule;
78 #( #rules )*
79 }
80
81 pub use self::visible::*;
82 }
83
84 ::pest::state(input, |state| {
85 match rule {
86 #patterns
87 }
88 })
89 }
90 }
91 };
92
93 quote! {
94 #include_fix
95 #rule_enum
96 #parser_impl
97 }
98 }
99
100 // Note: All builtin rules should be validated as pest builtins in meta/src/validator.rs.
101 // Some should also be keywords.
generate_builtin_rules() -> Vec<(&'static str, TokenStream)>102 fn generate_builtin_rules() -> Vec<(&'static str, TokenStream)> {
103 let mut builtins = Vec::new();
104
105 insert_builtin!(builtins, ANY, state.skip(1));
106 insert_builtin!(
107 builtins,
108 EOI,
109 state.rule(Rule::EOI, |state| state.end_of_input())
110 );
111 insert_builtin!(builtins, SOI, state.start_of_input());
112 insert_builtin!(builtins, PEEK, state.stack_peek());
113 insert_builtin!(builtins, PEEK_ALL, state.stack_match_peek());
114 insert_builtin!(builtins, POP, state.stack_pop());
115 insert_builtin!(builtins, POP_ALL, state.stack_match_pop());
116 insert_builtin!(builtins, DROP, state.stack_drop());
117
118 insert_builtin!(builtins, ASCII_DIGIT, state.match_range('0'..'9'));
119 insert_builtin!(builtins, ASCII_NONZERO_DIGIT, state.match_range('1'..'9'));
120 insert_builtin!(builtins, ASCII_BIN_DIGIT, state.match_range('0'..'1'));
121 insert_builtin!(builtins, ASCII_OCT_DIGIT, state.match_range('0'..'7'));
122 insert_builtin!(
123 builtins,
124 ASCII_HEX_DIGIT,
125 state
126 .match_range('0'..'9')
127 .or_else(|state| state.match_range('a'..'f'))
128 .or_else(|state| state.match_range('A'..'F'))
129 );
130 insert_builtin!(builtins, ASCII_ALPHA_LOWER, state.match_range('a'..'z'));
131 insert_builtin!(builtins, ASCII_ALPHA_UPPER, state.match_range('A'..'Z'));
132 insert_builtin!(
133 builtins,
134 ASCII_ALPHA,
135 state
136 .match_range('a'..'z')
137 .or_else(|state| state.match_range('A'..'Z'))
138 );
139 insert_builtin!(
140 builtins,
141 ASCII_ALPHANUMERIC,
142 state
143 .match_range('a'..'z')
144 .or_else(|state| state.match_range('A'..'Z'))
145 .or_else(|state| state.match_range('0'..'9'))
146 );
147 insert_builtin!(builtins, ASCII, state.match_range('\x00'..'\x7f'));
148 insert_builtin!(
149 builtins,
150 NEWLINE,
151 state
152 .match_string("\n")
153 .or_else(|state| state.match_string("\r\n"))
154 .or_else(|state| state.match_string("\r"))
155 );
156
157 let box_ty = box_type();
158
159 for property in unicode_property_names() {
160 let property_ident: Ident = syn::parse_str(property).unwrap();
161 // insert manually for #property substitution
162 builtins.push((property, quote! {
163 #[inline]
164 #[allow(dead_code, non_snake_case, unused_variables)]
165 fn #property_ident(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
166 state.match_char_by(::pest::unicode::#property_ident)
167 }
168 }));
169 }
170 builtins
171 }
172
173 // Needed because Cargo doesn't watch for changes in grammars.
generate_include(name: &Ident, path: &str) -> TokenStream174 fn generate_include(name: &Ident, path: &str) -> TokenStream {
175 let const_name = format_ident!("_PEST_GRAMMAR_{}", name);
176 // Need to make this relative to the current directory since the path to the file
177 // is derived from the CARGO_MANIFEST_DIR environment variable
178 let mut current_dir = std::env::current_dir().expect("Unable to get current directory");
179 current_dir.push(path);
180 let relative_path = current_dir.to_str().expect("path contains invalid unicode");
181 quote! {
182 #[allow(non_upper_case_globals)]
183 const #const_name: &'static str = include_str!(#relative_path);
184 }
185 }
186
generate_enum(rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bool) -> TokenStream187 fn generate_enum(rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bool) -> TokenStream {
188 let rules = rules.iter().map(|rule| {
189 let rule_name = format_ident!("r#{}", rule.name);
190
191 match doc_comment.line_docs.get(&rule.name) {
192 Some(doc) => quote! {
193 #[doc = #doc]
194 #rule_name
195 },
196 None => quote! {
197 #rule_name
198 },
199 }
200 });
201
202 let grammar_doc = &doc_comment.grammar_doc;
203 if uses_eoi {
204 quote! {
205 #[doc = #grammar_doc]
206 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
207 #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
208 pub enum Rule {
209 EOI,
210 #( #rules ),*
211 }
212 }
213 } else {
214 quote! {
215 #[doc = #grammar_doc]
216 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
217 #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
218 pub enum Rule {
219 #( #rules ),*
220 }
221 }
222 }
223 }
224
generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream225 fn generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream {
226 let mut rules: Vec<TokenStream> = rules
227 .iter()
228 .map(|rule| {
229 let rule = format_ident!("r#{}", rule.name);
230
231 quote! {
232 Rule::#rule => rules::#rule(state)
233 }
234 })
235 .collect();
236
237 if uses_eoi {
238 rules.push(quote! {
239 Rule::EOI => rules::EOI(state)
240 });
241 }
242
243 quote! {
244 #( #rules ),*
245 }
246 }
247
generate_rule(rule: OptimizedRule) -> TokenStream248 fn generate_rule(rule: OptimizedRule) -> TokenStream {
249 let name = format_ident!("r#{}", rule.name);
250 let expr = if rule.ty == RuleType::Atomic || rule.ty == RuleType::CompoundAtomic {
251 generate_expr_atomic(rule.expr)
252 } else if rule.name == "WHITESPACE" || rule.name == "COMMENT" {
253 let atomic = generate_expr_atomic(rule.expr);
254
255 quote! {
256 state.atomic(::pest::Atomicity::Atomic, |state| {
257 #atomic
258 })
259 }
260 } else {
261 generate_expr(rule.expr)
262 };
263
264 let box_ty = box_type();
265
266 match rule.ty {
267 RuleType::Normal => quote! {
268 #[inline]
269 #[allow(non_snake_case, unused_variables)]
270 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
271 state.rule(Rule::#name, |state| {
272 #expr
273 })
274 }
275 },
276 RuleType::Silent => quote! {
277 #[inline]
278 #[allow(non_snake_case, unused_variables)]
279 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
280 #expr
281 }
282 },
283 RuleType::Atomic => quote! {
284 #[inline]
285 #[allow(non_snake_case, unused_variables)]
286 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
287 state.rule(Rule::#name, |state| {
288 state.atomic(::pest::Atomicity::Atomic, |state| {
289 #expr
290 })
291 })
292 }
293 },
294 RuleType::CompoundAtomic => quote! {
295 #[inline]
296 #[allow(non_snake_case, unused_variables)]
297 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
298 state.atomic(::pest::Atomicity::CompoundAtomic, |state| {
299 state.rule(Rule::#name, |state| {
300 #expr
301 })
302 })
303 }
304 },
305 RuleType::NonAtomic => quote! {
306 #[inline]
307 #[allow(non_snake_case, unused_variables)]
308 pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
309 state.atomic(::pest::Atomicity::NonAtomic, |state| {
310 state.rule(Rule::#name, |state| {
311 #expr
312 })
313 })
314 }
315 },
316 }
317 }
318
generate_skip(rules: &[OptimizedRule]) -> TokenStream319 fn generate_skip(rules: &[OptimizedRule]) -> TokenStream {
320 let whitespace = rules.iter().any(|rule| rule.name == "WHITESPACE");
321 let comment = rules.iter().any(|rule| rule.name == "COMMENT");
322
323 match (whitespace, comment) {
324 (false, false) => generate_rule!(skip, Ok(state)),
325 (true, false) => generate_rule!(
326 skip,
327 if state.atomicity() == ::pest::Atomicity::NonAtomic {
328 state.repeat(|state| super::visible::WHITESPACE(state))
329 } else {
330 Ok(state)
331 }
332 ),
333 (false, true) => generate_rule!(
334 skip,
335 if state.atomicity() == ::pest::Atomicity::NonAtomic {
336 state.repeat(|state| super::visible::COMMENT(state))
337 } else {
338 Ok(state)
339 }
340 ),
341 (true, true) => generate_rule!(
342 skip,
343 if state.atomicity() == ::pest::Atomicity::NonAtomic {
344 state.sequence(|state| {
345 state
346 .repeat(|state| super::visible::WHITESPACE(state))
347 .and_then(|state| {
348 state.repeat(|state| {
349 state.sequence(|state| {
350 super::visible::COMMENT(state).and_then(|state| {
351 state.repeat(|state| super::visible::WHITESPACE(state))
352 })
353 })
354 })
355 })
356 })
357 } else {
358 Ok(state)
359 }
360 ),
361 }
362 }
363
generate_expr(expr: OptimizedExpr) -> TokenStream364 fn generate_expr(expr: OptimizedExpr) -> TokenStream {
365 match expr {
366 OptimizedExpr::Str(string) => {
367 quote! {
368 state.match_string(#string)
369 }
370 }
371 OptimizedExpr::Insens(string) => {
372 quote! {
373 state.match_insensitive(#string)
374 }
375 }
376 OptimizedExpr::Range(start, end) => {
377 let start = start.chars().next().unwrap();
378 let end = end.chars().next().unwrap();
379
380 quote! {
381 state.match_range(#start..#end)
382 }
383 }
384 OptimizedExpr::Ident(ident) => {
385 let ident = format_ident!("r#{}", ident);
386 quote! { self::#ident(state) }
387 }
388 OptimizedExpr::PeekSlice(start, end_) => {
389 let end = QuoteOption(end_);
390 quote! {
391 state.stack_match_peek_slice(#start, #end, ::pest::MatchDir::BottomToTop)
392 }
393 }
394 OptimizedExpr::PosPred(expr) => {
395 let expr = generate_expr(*expr);
396
397 quote! {
398 state.lookahead(true, |state| {
399 #expr
400 })
401 }
402 }
403 OptimizedExpr::NegPred(expr) => {
404 let expr = generate_expr(*expr);
405
406 quote! {
407 state.lookahead(false, |state| {
408 #expr
409 })
410 }
411 }
412 OptimizedExpr::Seq(lhs, rhs) => {
413 let head = generate_expr(*lhs);
414 let mut tail = vec![];
415 let mut current = *rhs;
416
417 while let OptimizedExpr::Seq(lhs, rhs) = current {
418 tail.push(generate_expr(*lhs));
419 current = *rhs;
420 }
421 tail.push(generate_expr(current));
422
423 quote! {
424 state.sequence(|state| {
425 #head
426 #(
427 .and_then(|state| {
428 super::hidden::skip(state)
429 }).and_then(|state| {
430 #tail
431 })
432 )*
433 })
434 }
435 }
436 OptimizedExpr::Choice(lhs, rhs) => {
437 let head = generate_expr(*lhs);
438 let mut tail = vec![];
439 let mut current = *rhs;
440
441 while let OptimizedExpr::Choice(lhs, rhs) = current {
442 tail.push(generate_expr(*lhs));
443 current = *rhs;
444 }
445 tail.push(generate_expr(current));
446
447 quote! {
448 #head
449 #(
450 .or_else(|state| {
451 #tail
452 })
453 )*
454 }
455 }
456 OptimizedExpr::Opt(expr) => {
457 let expr = generate_expr(*expr);
458
459 quote! {
460 state.optional(|state| {
461 #expr
462 })
463 }
464 }
465 OptimizedExpr::Rep(expr) => {
466 let expr = generate_expr(*expr);
467
468 quote! {
469 state.sequence(|state| {
470 state.optional(|state| {
471 #expr.and_then(|state| {
472 state.repeat(|state| {
473 state.sequence(|state| {
474 super::hidden::skip(
475 state
476 ).and_then(|state| {
477 #expr
478 })
479 })
480 })
481 })
482 })
483 })
484 }
485 }
486 OptimizedExpr::Skip(strings) => {
487 quote! {
488 let strings = [#(#strings),*];
489
490 state.skip_until(&strings)
491 }
492 }
493 OptimizedExpr::Push(expr) => {
494 let expr = generate_expr(*expr);
495
496 quote! {
497 state.stack_push(|state| #expr)
498 }
499 }
500 OptimizedExpr::RestoreOnErr(expr) => {
501 let expr = generate_expr(*expr);
502
503 quote! {
504 state.restore_on_err(|state| #expr)
505 }
506 }
507 }
508 }
509
generate_expr_atomic(expr: OptimizedExpr) -> TokenStream510 fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
511 match expr {
512 OptimizedExpr::Str(string) => {
513 quote! {
514 state.match_string(#string)
515 }
516 }
517 OptimizedExpr::Insens(string) => {
518 quote! {
519 state.match_insensitive(#string)
520 }
521 }
522 OptimizedExpr::Range(start, end) => {
523 let start = start.chars().next().unwrap();
524 let end = end.chars().next().unwrap();
525
526 quote! {
527 state.match_range(#start..#end)
528 }
529 }
530 OptimizedExpr::Ident(ident) => {
531 let ident = format_ident!("r#{}", ident);
532 quote! { self::#ident(state) }
533 }
534 OptimizedExpr::PeekSlice(start, end_) => {
535 let end = QuoteOption(end_);
536 quote! {
537 state.stack_match_peek_slice(#start, #end, ::pest::MatchDir::BottomToTop)
538 }
539 }
540 OptimizedExpr::PosPred(expr) => {
541 let expr = generate_expr_atomic(*expr);
542
543 quote! {
544 state.lookahead(true, |state| {
545 #expr
546 })
547 }
548 }
549 OptimizedExpr::NegPred(expr) => {
550 let expr = generate_expr_atomic(*expr);
551
552 quote! {
553 state.lookahead(false, |state| {
554 #expr
555 })
556 }
557 }
558 OptimizedExpr::Seq(lhs, rhs) => {
559 let head = generate_expr_atomic(*lhs);
560 let mut tail = vec![];
561 let mut current = *rhs;
562
563 while let OptimizedExpr::Seq(lhs, rhs) = current {
564 tail.push(generate_expr_atomic(*lhs));
565 current = *rhs;
566 }
567 tail.push(generate_expr_atomic(current));
568
569 quote! {
570 state.sequence(|state| {
571 #head
572 #(
573 .and_then(|state| {
574 #tail
575 })
576 )*
577 })
578 }
579 }
580 OptimizedExpr::Choice(lhs, rhs) => {
581 let head = generate_expr_atomic(*lhs);
582 let mut tail = vec![];
583 let mut current = *rhs;
584
585 while let OptimizedExpr::Choice(lhs, rhs) = current {
586 tail.push(generate_expr_atomic(*lhs));
587 current = *rhs;
588 }
589 tail.push(generate_expr_atomic(current));
590
591 quote! {
592 #head
593 #(
594 .or_else(|state| {
595 #tail
596 })
597 )*
598 }
599 }
600 OptimizedExpr::Opt(expr) => {
601 let expr = generate_expr_atomic(*expr);
602
603 quote! {
604 state.optional(|state| {
605 #expr
606 })
607 }
608 }
609 OptimizedExpr::Rep(expr) => {
610 let expr = generate_expr_atomic(*expr);
611
612 quote! {
613 state.repeat(|state| {
614 #expr
615 })
616 }
617 }
618 OptimizedExpr::Skip(strings) => {
619 quote! {
620 let strings = [#(#strings),*];
621
622 state.skip_until(&strings)
623 }
624 }
625 OptimizedExpr::Push(expr) => {
626 let expr = generate_expr_atomic(*expr);
627
628 quote! {
629 state.stack_push(|state| #expr)
630 }
631 }
632 OptimizedExpr::RestoreOnErr(expr) => {
633 let expr = generate_expr_atomic(*expr);
634
635 quote! {
636 state.restore_on_err(|state| #expr)
637 }
638 }
639 }
640 }
641
642 struct QuoteOption<T>(Option<T>);
643
644 impl<T: ToTokens> ToTokens for QuoteOption<T> {
to_tokens(&self, tokens: &mut TokenStream)645 fn to_tokens(&self, tokens: &mut TokenStream) {
646 let option = option_type();
647 tokens.append_all(match self.0 {
648 Some(ref t) => quote! { #option::Some(#t) },
649 None => quote! { #option::None },
650 });
651 }
652 }
653
box_type() -> TokenStream654 fn box_type() -> TokenStream {
655 #[cfg(feature = "std")]
656 quote! { ::std::boxed::Box }
657
658 #[cfg(not(feature = "std"))]
659 quote! { ::alloc::boxed::Box }
660 }
661
result_type() -> TokenStream662 fn result_type() -> TokenStream {
663 #[cfg(feature = "std")]
664 quote! { ::std::result::Result }
665
666 #[cfg(not(feature = "std"))]
667 quote! { ::core::result::Result }
668 }
669
option_type() -> TokenStream670 fn option_type() -> TokenStream {
671 #[cfg(feature = "std")]
672 quote! { ::std::option::Option }
673
674 #[cfg(not(feature = "std"))]
675 quote! { ::core::option::Option }
676 }
677
678 #[cfg(test)]
679 mod tests {
680 use super::*;
681
682 use proc_macro2::Span;
683 use std::collections::HashMap;
684
685 #[test]
rule_enum_simple()686 fn rule_enum_simple() {
687 let rules = vec![OptimizedRule {
688 name: "f".to_owned(),
689 ty: RuleType::Normal,
690 expr: OptimizedExpr::Ident("g".to_owned()),
691 }];
692
693 let mut line_docs = HashMap::new();
694 line_docs.insert("f".to_owned(), "This is rule comment".to_owned());
695
696 let doc_comment = &DocComment {
697 grammar_doc: "Rule doc\nhello".to_owned(),
698 line_docs,
699 };
700
701 assert_eq!(
702 generate_enum(&rules, doc_comment, false).to_string(),
703 quote! {
704 #[doc = "Rule doc\nhello"]
705 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
706 #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
707 pub enum Rule {
708 #[doc = "This is rule comment"]
709 r#f
710 }
711 }
712 .to_string()
713 );
714 }
715
716 #[test]
sequence()717 fn sequence() {
718 let expr = OptimizedExpr::Seq(
719 Box::new(OptimizedExpr::Str("a".to_owned())),
720 Box::new(OptimizedExpr::Seq(
721 Box::new(OptimizedExpr::Str("b".to_owned())),
722 Box::new(OptimizedExpr::Seq(
723 Box::new(OptimizedExpr::Str("c".to_owned())),
724 Box::new(OptimizedExpr::Str("d".to_owned())),
725 )),
726 )),
727 );
728
729 assert_eq!(
730 generate_expr(expr).to_string(),
731 quote! {
732 state.sequence(|state| {
733 state.match_string("a").and_then(|state| {
734 super::hidden::skip(state)
735 }).and_then(|state| {
736 state.match_string("b")
737 }).and_then(|state| {
738 super::hidden::skip(state)
739 }).and_then(|state| {
740 state.match_string("c")
741 }).and_then(|state| {
742 super::hidden::skip(state)
743 }).and_then(|state| {
744 state.match_string("d")
745 })
746 })
747 }
748 .to_string()
749 );
750 }
751
752 #[test]
sequence_atomic()753 fn sequence_atomic() {
754 let expr = OptimizedExpr::Seq(
755 Box::new(OptimizedExpr::Str("a".to_owned())),
756 Box::new(OptimizedExpr::Seq(
757 Box::new(OptimizedExpr::Str("b".to_owned())),
758 Box::new(OptimizedExpr::Seq(
759 Box::new(OptimizedExpr::Str("c".to_owned())),
760 Box::new(OptimizedExpr::Str("d".to_owned())),
761 )),
762 )),
763 );
764
765 assert_eq!(
766 generate_expr_atomic(expr).to_string(),
767 quote! {
768 state.sequence(|state| {
769 state.match_string("a").and_then(|state| {
770 state.match_string("b")
771 }).and_then(|state| {
772 state.match_string("c")
773 }).and_then(|state| {
774 state.match_string("d")
775 })
776 })
777 }
778 .to_string()
779 );
780 }
781
782 #[test]
choice()783 fn choice() {
784 let expr = OptimizedExpr::Choice(
785 Box::new(OptimizedExpr::Str("a".to_owned())),
786 Box::new(OptimizedExpr::Choice(
787 Box::new(OptimizedExpr::Str("b".to_owned())),
788 Box::new(OptimizedExpr::Choice(
789 Box::new(OptimizedExpr::Str("c".to_owned())),
790 Box::new(OptimizedExpr::Str("d".to_owned())),
791 )),
792 )),
793 );
794
795 assert_eq!(
796 generate_expr(expr).to_string(),
797 quote! {
798 state.match_string("a").or_else(|state| {
799 state.match_string("b")
800 }).or_else(|state| {
801 state.match_string("c")
802 }).or_else(|state| {
803 state.match_string("d")
804 })
805 }
806 .to_string()
807 );
808 }
809
810 #[test]
choice_atomic()811 fn choice_atomic() {
812 let expr = OptimizedExpr::Choice(
813 Box::new(OptimizedExpr::Str("a".to_owned())),
814 Box::new(OptimizedExpr::Choice(
815 Box::new(OptimizedExpr::Str("b".to_owned())),
816 Box::new(OptimizedExpr::Choice(
817 Box::new(OptimizedExpr::Str("c".to_owned())),
818 Box::new(OptimizedExpr::Str("d".to_owned())),
819 )),
820 )),
821 );
822
823 assert_eq!(
824 generate_expr_atomic(expr).to_string(),
825 quote! {
826 state.match_string("a").or_else(|state| {
827 state.match_string("b")
828 }).or_else(|state| {
829 state.match_string("c")
830 }).or_else(|state| {
831 state.match_string("d")
832 })
833 }
834 .to_string()
835 );
836 }
837
838 #[test]
skip()839 fn skip() {
840 let expr = OptimizedExpr::Skip(vec!["a".to_owned(), "b".to_owned()]);
841
842 assert_eq!(
843 generate_expr_atomic(expr).to_string(),
844 quote! {
845 let strings = ["a", "b"];
846
847 state.skip_until(&strings)
848 }
849 .to_string()
850 );
851 }
852
853 #[test]
expr_complex()854 fn expr_complex() {
855 let expr = OptimizedExpr::Choice(
856 Box::new(OptimizedExpr::Ident("a".to_owned())),
857 Box::new(OptimizedExpr::Seq(
858 Box::new(OptimizedExpr::Range("a".to_owned(), "b".to_owned())),
859 Box::new(OptimizedExpr::Seq(
860 Box::new(OptimizedExpr::NegPred(Box::new(OptimizedExpr::Rep(
861 Box::new(OptimizedExpr::Insens("b".to_owned())),
862 )))),
863 Box::new(OptimizedExpr::PosPred(Box::new(OptimizedExpr::Opt(
864 Box::new(OptimizedExpr::Rep(Box::new(OptimizedExpr::Choice(
865 Box::new(OptimizedExpr::Str("c".to_owned())),
866 Box::new(OptimizedExpr::Str("d".to_owned())),
867 )))),
868 )))),
869 )),
870 )),
871 );
872
873 let sequence = quote! {
874 state.sequence(|state| {
875 super::hidden::skip(state).and_then(
876 |state| {
877 state.match_insensitive("b")
878 }
879 )
880 })
881 };
882 let repeat = quote! {
883 state.repeat(|state| {
884 state.sequence(|state| {
885 super::hidden::skip(state).and_then(|state| {
886 state.match_string("c")
887 .or_else(|state| {
888 state.match_string("d")
889 })
890 })
891 })
892 })
893 };
894 assert_eq!(
895 generate_expr(expr).to_string(),
896 quote! {
897 self::r#a(state).or_else(|state| {
898 state.sequence(|state| {
899 state.match_range('a'..'b').and_then(|state| {
900 super::hidden::skip(state)
901 }).and_then(|state| {
902 state.lookahead(false, |state| {
903 state.sequence(|state| {
904 state.optional(|state| {
905 state.match_insensitive(
906 "b"
907 ).and_then(|state| {
908 state.repeat(|state| {
909 #sequence
910 })
911 })
912 })
913 })
914 })
915 }).and_then(|state| {
916 super::hidden::skip(state)
917 }).and_then(|state| {
918 state.lookahead(true, |state| {
919 state.optional(|state| {
920 state.sequence(|state| {
921 state.optional(|state| {
922 state.match_string("c")
923 .or_else(|state| {
924 state.match_string("d")
925 }).and_then(|state| {
926 #repeat
927 })
928 })
929 })
930 })
931 })
932 })
933 })
934 })
935 }
936 .to_string()
937 );
938 }
939
940 #[test]
expr_complex_atomic()941 fn expr_complex_atomic() {
942 let expr = OptimizedExpr::Choice(
943 Box::new(OptimizedExpr::Ident("a".to_owned())),
944 Box::new(OptimizedExpr::Seq(
945 Box::new(OptimizedExpr::Range("a".to_owned(), "b".to_owned())),
946 Box::new(OptimizedExpr::Seq(
947 Box::new(OptimizedExpr::NegPred(Box::new(OptimizedExpr::Rep(
948 Box::new(OptimizedExpr::Insens("b".to_owned())),
949 )))),
950 Box::new(OptimizedExpr::PosPred(Box::new(OptimizedExpr::Opt(
951 Box::new(OptimizedExpr::Rep(Box::new(OptimizedExpr::Choice(
952 Box::new(OptimizedExpr::Str("c".to_owned())),
953 Box::new(OptimizedExpr::Str("d".to_owned())),
954 )))),
955 )))),
956 )),
957 )),
958 );
959
960 assert_eq!(
961 generate_expr_atomic(expr).to_string(),
962 quote! {
963 self::r#a(state).or_else(|state| {
964 state.sequence(|state| {
965 state.match_range('a'..'b').and_then(|state| {
966 state.lookahead(false, |state| {
967 state.repeat(|state| {
968 state.match_insensitive("b")
969 })
970 })
971 }).and_then(|state| {
972 state.lookahead(true, |state| {
973 state.optional(|state| {
974 state.repeat(|state| {
975 state.match_string("c")
976 .or_else(|state| {
977 state.match_string("d")
978 })
979 })
980 })
981 })
982 })
983 })
984 })
985 }
986 .to_string()
987 );
988 }
989
990 #[test]
test_generate_complete()991 fn test_generate_complete() {
992 let name = Ident::new("MyParser", Span::call_site());
993 let generics = Generics::default();
994
995 let rules = vec![
996 OptimizedRule {
997 name: "a".to_owned(),
998 ty: RuleType::Silent,
999 expr: OptimizedExpr::Str("b".to_owned()),
1000 },
1001 OptimizedRule {
1002 name: "if".to_owned(),
1003 ty: RuleType::Silent,
1004 expr: OptimizedExpr::Ident("a".to_owned()),
1005 },
1006 ];
1007
1008 let mut line_docs = HashMap::new();
1009 line_docs.insert("if".to_owned(), "If statement".to_owned());
1010
1011 let doc_comment = &DocComment {
1012 line_docs,
1013 grammar_doc: "This is Rule doc\nThis is second line".to_owned(),
1014 };
1015
1016 let defaults = vec!["ANY"];
1017 let result = result_type();
1018 let box_ty = box_type();
1019 let mut current_dir = std::env::current_dir().expect("Unable to get current directory");
1020 current_dir.push("test.pest");
1021 let test_path = current_dir.to_str().expect("path contains invalid unicode");
1022 assert_eq!(
1023 generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, doc_comment, true).to_string(),
1024 quote! {
1025 #[allow(non_upper_case_globals)]
1026 const _PEST_GRAMMAR_MyParser: &'static str = include_str!(#test_path);
1027
1028 #[doc = "This is Rule doc\nThis is second line"]
1029 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
1030 #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
1031 pub enum Rule {
1032 r#a,
1033 #[doc = "If statement"]
1034 r#if
1035 }
1036
1037 #[allow(clippy::all)]
1038 impl ::pest::Parser<Rule> for MyParser {
1039 fn parse<'i>(
1040 rule: Rule,
1041 input: &'i str
1042 ) -> #result<
1043 ::pest::iterators::Pairs<'i, Rule>,
1044 ::pest::error::Error<Rule>
1045 > {
1046 mod rules {
1047 #![allow(clippy::upper_case_acronyms)]
1048 pub mod hidden {
1049 use super::super::Rule;
1050
1051 #[inline]
1052 #[allow(dead_code, non_snake_case, unused_variables)]
1053 pub fn skip(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
1054 Ok(state)
1055 }
1056 }
1057
1058 pub mod visible {
1059 use super::super::Rule;
1060
1061 #[inline]
1062 #[allow(non_snake_case, unused_variables)]
1063 pub fn r#a(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
1064 state.match_string("b")
1065 }
1066
1067 #[inline]
1068 #[allow(non_snake_case, unused_variables)]
1069 pub fn r#if(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
1070 self::r#a(state)
1071 }
1072
1073 #[inline]
1074 #[allow(dead_code, non_snake_case, unused_variables)]
1075 pub fn ANY(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> {
1076 state.skip(1)
1077 }
1078 }
1079
1080 pub use self::visible::*;
1081 }
1082
1083 ::pest::state(input, |state| {
1084 match rule {
1085 Rule::r#a => rules::r#a(state),
1086 Rule::r#if => rules::r#if(state)
1087 }
1088 })
1089 }
1090 }
1091 }.to_string()
1092 );
1093 }
1094 }
1095