1import ast 2from dataclasses import field, dataclass 3import re 4from typing import Any, Dict, IO, Optional, List, Text, Tuple, Set 5from enum import Enum 6 7from pegen import grammar 8from pegen.grammar import ( 9 Alt, 10 Cut, 11 Gather, 12 GrammarVisitor, 13 Group, 14 Lookahead, 15 NamedItem, 16 NameLeaf, 17 NegativeLookahead, 18 Opt, 19 PositiveLookahead, 20 Repeat0, 21 Repeat1, 22 Rhs, 23 Rule, 24 StringLeaf, 25) 26from pegen.parser_generator import ParserGenerator 27 28 29EXTENSION_PREFIX = """\ 30#include "pegen.h" 31 32#if defined(Py_DEBUG) && defined(Py_BUILD_CORE) 33extern int Py_DebugFlag; 34#define D(x) if (Py_DebugFlag) x; 35#else 36#define D(x) 37#endif 38 39""" 40 41 42EXTENSION_SUFFIX = """ 43void * 44_PyPegen_parse(Parser *p) 45{ 46 // Initialize keywords 47 p->keywords = reserved_keywords; 48 p->n_keyword_lists = n_keyword_lists; 49 50 return start_rule(p); 51} 52""" 53 54 55class NodeTypes(Enum): 56 NAME_TOKEN = 0 57 NUMBER_TOKEN = 1 58 STRING_TOKEN = 2 59 GENERIC_TOKEN = 3 60 KEYWORD = 4 61 SOFT_KEYWORD = 5 62 CUT_OPERATOR = 6 63 64 65BASE_NODETYPES = { 66 "NAME": NodeTypes.NAME_TOKEN, 67 "NUMBER": NodeTypes.NUMBER_TOKEN, 68 "STRING": NodeTypes.STRING_TOKEN, 69} 70 71 72@dataclass 73class FunctionCall: 74 function: str 75 arguments: List[Any] = field(default_factory=list) 76 assigned_variable: Optional[str] = None 77 return_type: Optional[str] = None 78 nodetype: Optional[NodeTypes] = None 79 force_true: bool = False 80 comment: Optional[str] = None 81 82 def __str__(self) -> str: 83 parts = [] 84 parts.append(self.function) 85 if self.arguments: 86 parts.append(f"({', '.join(map(str, self.arguments))})") 87 if self.force_true: 88 parts.append(", 1") 89 if self.assigned_variable: 90 parts = ["(", self.assigned_variable, " = ", *parts, ")"] 91 if self.comment: 92 parts.append(f" // {self.comment}") 93 return "".join(parts) 94 95 96class CCallMakerVisitor(GrammarVisitor): 97 def __init__( 98 self, 99 parser_generator: ParserGenerator, 100 exact_tokens: Dict[str, int], 101 non_exact_tokens: Set[str], 102 ): 103 self.gen = parser_generator 104 self.exact_tokens = exact_tokens 105 self.non_exact_tokens = non_exact_tokens 106 self.cache: Dict[Any, FunctionCall] = {} 107 self.keyword_cache: Dict[str, int] = {} 108 self.soft_keywords: Set[str] = set() 109 110 def keyword_helper(self, keyword: str) -> FunctionCall: 111 if keyword not in self.keyword_cache: 112 self.keyword_cache[keyword] = self.gen.keyword_type() 113 return FunctionCall( 114 assigned_variable="_keyword", 115 function="_PyPegen_expect_token", 116 arguments=["p", self.keyword_cache[keyword]], 117 return_type="Token *", 118 nodetype=NodeTypes.KEYWORD, 119 comment=f"token='{keyword}'", 120 ) 121 122 def soft_keyword_helper(self, value: str) -> FunctionCall: 123 self.soft_keywords.add(value.replace('"', "")) 124 return FunctionCall( 125 assigned_variable="_keyword", 126 function="_PyPegen_expect_soft_keyword", 127 arguments=["p", value], 128 return_type="expr_ty", 129 nodetype=NodeTypes.SOFT_KEYWORD, 130 comment=f"soft_keyword='{value}'", 131 ) 132 133 def visit_NameLeaf(self, node: NameLeaf) -> FunctionCall: 134 name = node.value 135 if name in self.non_exact_tokens: 136 if name in BASE_NODETYPES: 137 return FunctionCall( 138 assigned_variable=f"{name.lower()}_var", 139 function=f"_PyPegen_{name.lower()}_token", 140 arguments=["p"], 141 nodetype=BASE_NODETYPES[name], 142 return_type="expr_ty", 143 comment=name, 144 ) 145 return FunctionCall( 146 assigned_variable=f"{name.lower()}_var", 147 function=f"_PyPegen_expect_token", 148 arguments=["p", name], 149 nodetype=NodeTypes.GENERIC_TOKEN, 150 return_type="Token *", 151 comment=f"token='{name}'", 152 ) 153 154 type = None 155 rule = self.gen.all_rules.get(name.lower()) 156 if rule is not None: 157 type = "asdl_seq *" if rule.is_loop() or rule.is_gather() else rule.type 158 159 return FunctionCall( 160 assigned_variable=f"{name}_var", 161 function=f"{name}_rule", 162 arguments=["p"], 163 return_type=type, 164 comment=f"{node}", 165 ) 166 167 def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall: 168 val = ast.literal_eval(node.value) 169 if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword 170 if node.value.endswith("'"): 171 return self.keyword_helper(val) 172 else: 173 return self.soft_keyword_helper(node.value) 174 else: 175 assert val in self.exact_tokens, f"{node.value} is not a known literal" 176 type = self.exact_tokens[val] 177 return FunctionCall( 178 assigned_variable="_literal", 179 function=f"_PyPegen_expect_token", 180 arguments=["p", type], 181 nodetype=NodeTypes.GENERIC_TOKEN, 182 return_type="Token *", 183 comment=f"token='{val}'", 184 ) 185 186 def visit_Rhs(self, node: Rhs) -> FunctionCall: 187 def can_we_inline(node: Rhs) -> int: 188 if len(node.alts) != 1 or len(node.alts[0].items) != 1: 189 return False 190 # If the alternative has an action we cannot inline 191 if getattr(node.alts[0], "action", None) is not None: 192 return False 193 return True 194 195 if node in self.cache: 196 return self.cache[node] 197 if can_we_inline(node): 198 self.cache[node] = self.generate_call(node.alts[0].items[0]) 199 else: 200 name = self.gen.name_node(node) 201 self.cache[node] = FunctionCall( 202 assigned_variable=f"{name}_var", 203 function=f"{name}_rule", 204 arguments=["p"], 205 comment=f"{node}", 206 ) 207 return self.cache[node] 208 209 def visit_NamedItem(self, node: NamedItem) -> FunctionCall: 210 call = self.generate_call(node.item) 211 if node.name: 212 call.assigned_variable = node.name 213 return call 214 215 def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall: 216 call = self.generate_call(node.node) 217 if call.nodetype == NodeTypes.NAME_TOKEN: 218 return FunctionCall( 219 function=f"_PyPegen_lookahead_with_name", 220 arguments=[positive, call.function, *call.arguments], 221 return_type="int", 222 ) 223 elif call.nodetype == NodeTypes.SOFT_KEYWORD: 224 return FunctionCall( 225 function=f"_PyPegen_lookahead_with_string", 226 arguments=[positive, call.function, *call.arguments], 227 return_type="int", 228 ) 229 elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}: 230 return FunctionCall( 231 function=f"_PyPegen_lookahead_with_int", 232 arguments=[positive, call.function, *call.arguments], 233 return_type="int", 234 comment=f"token={node.node}", 235 ) 236 else: 237 return FunctionCall( 238 function=f"_PyPegen_lookahead", 239 arguments=[positive, call.function, *call.arguments], 240 return_type="int", 241 ) 242 243 def visit_PositiveLookahead(self, node: PositiveLookahead) -> FunctionCall: 244 return self.lookahead_call_helper(node, 1) 245 246 def visit_NegativeLookahead(self, node: NegativeLookahead) -> FunctionCall: 247 return self.lookahead_call_helper(node, 0) 248 249 def visit_Opt(self, node: Opt) -> FunctionCall: 250 call = self.generate_call(node.node) 251 return FunctionCall( 252 assigned_variable="_opt_var", 253 function=call.function, 254 arguments=call.arguments, 255 force_true=True, 256 comment=f"{node}", 257 ) 258 259 def visit_Repeat0(self, node: Repeat0) -> FunctionCall: 260 if node in self.cache: 261 return self.cache[node] 262 name = self.gen.name_loop(node.node, False) 263 self.cache[node] = FunctionCall( 264 assigned_variable=f"{name}_var", 265 function=f"{name}_rule", 266 arguments=["p"], 267 return_type="asdl_seq *", 268 comment=f"{node}", 269 ) 270 return self.cache[node] 271 272 def visit_Repeat1(self, node: Repeat1) -> FunctionCall: 273 if node in self.cache: 274 return self.cache[node] 275 name = self.gen.name_loop(node.node, True) 276 self.cache[node] = FunctionCall( 277 assigned_variable=f"{name}_var", 278 function=f"{name}_rule", 279 arguments=["p"], 280 return_type="asdl_seq *", 281 comment=f"{node}", 282 ) 283 return self.cache[node] 284 285 def visit_Gather(self, node: Gather) -> FunctionCall: 286 if node in self.cache: 287 return self.cache[node] 288 name = self.gen.name_gather(node) 289 self.cache[node] = FunctionCall( 290 assigned_variable=f"{name}_var", 291 function=f"{name}_rule", 292 arguments=["p"], 293 return_type="asdl_seq *", 294 comment=f"{node}", 295 ) 296 return self.cache[node] 297 298 def visit_Group(self, node: Group) -> FunctionCall: 299 return self.generate_call(node.rhs) 300 301 def visit_Cut(self, node: Cut) -> FunctionCall: 302 return FunctionCall( 303 assigned_variable="_cut_var", 304 return_type="int", 305 function="1", 306 nodetype=NodeTypes.CUT_OPERATOR, 307 ) 308 309 def generate_call(self, node: Any) -> FunctionCall: 310 return super().visit(node) 311 312 313class CParserGenerator(ParserGenerator, GrammarVisitor): 314 def __init__( 315 self, 316 grammar: grammar.Grammar, 317 tokens: Dict[int, str], 318 exact_tokens: Dict[str, int], 319 non_exact_tokens: Set[str], 320 file: Optional[IO[Text]], 321 debug: bool = False, 322 skip_actions: bool = False, 323 ): 324 super().__init__(grammar, tokens, file) 325 self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor( 326 self, exact_tokens, non_exact_tokens 327 ) 328 self._varname_counter = 0 329 self.debug = debug 330 self.skip_actions = skip_actions 331 332 def add_level(self) -> None: 333 self.print("D(p->level++);") 334 335 def remove_level(self) -> None: 336 self.print("D(p->level--);") 337 338 def add_return(self, ret_val: str) -> None: 339 self.remove_level() 340 self.print(f"return {ret_val};") 341 342 def unique_varname(self, name: str = "tmpvar") -> str: 343 new_var = name + "_" + str(self._varname_counter) 344 self._varname_counter += 1 345 return new_var 346 347 def call_with_errorcheck_return(self, call_text: str, returnval: str) -> None: 348 error_var = self.unique_varname() 349 self.print(f"int {error_var} = {call_text};") 350 self.print(f"if ({error_var}) {{") 351 with self.indent(): 352 self.add_return(returnval) 353 self.print("}") 354 355 def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None: 356 error_var = self.unique_varname() 357 self.print(f"int {error_var} = {call_text};") 358 self.print(f"if ({error_var}) {{") 359 with self.indent(): 360 self.print(f"goto {goto_target};") 361 self.print(f"}}") 362 363 def out_of_memory_return(self, expr: str, cleanup_code: Optional[str] = None,) -> None: 364 self.print(f"if ({expr}) {{") 365 with self.indent(): 366 if cleanup_code is not None: 367 self.print(cleanup_code) 368 self.print("p->error_indicator = 1;") 369 self.print("PyErr_NoMemory();") 370 self.add_return("NULL") 371 self.print(f"}}") 372 373 def out_of_memory_goto(self, expr: str, goto_target: str) -> None: 374 self.print(f"if ({expr}) {{") 375 with self.indent(): 376 self.print("PyErr_NoMemory();") 377 self.print(f"goto {goto_target};") 378 self.print(f"}}") 379 380 def generate(self, filename: str) -> None: 381 self.collect_todo() 382 self.print(f"// @generated by pegen.py from {filename}") 383 header = self.grammar.metas.get("header", EXTENSION_PREFIX) 384 if header: 385 self.print(header.rstrip("\n")) 386 subheader = self.grammar.metas.get("subheader", "") 387 if subheader: 388 self.print(subheader) 389 self._setup_keywords() 390 for i, (rulename, rule) in enumerate(self.todo.items(), 1000): 391 comment = " // Left-recursive" if rule.left_recursive else "" 392 self.print(f"#define {rulename}_type {i}{comment}") 393 self.print() 394 for rulename, rule in self.todo.items(): 395 if rule.is_loop() or rule.is_gather(): 396 type = "asdl_seq *" 397 elif rule.type: 398 type = rule.type + " " 399 else: 400 type = "void *" 401 self.print(f"static {type}{rulename}_rule(Parser *p);") 402 self.print() 403 while self.todo: 404 for rulename, rule in list(self.todo.items()): 405 del self.todo[rulename] 406 self.print() 407 if rule.left_recursive: 408 self.print("// Left-recursive") 409 self.visit(rule) 410 if self.skip_actions: 411 mode = 0 412 else: 413 mode = int(self.rules["start"].type == "mod_ty") if "start" in self.rules else 1 414 if mode == 1 and self.grammar.metas.get("bytecode"): 415 mode += 1 416 modulename = self.grammar.metas.get("modulename", "parse") 417 trailer = self.grammar.metas.get("trailer", EXTENSION_SUFFIX) 418 if trailer: 419 self.print(trailer.rstrip("\n") % dict(mode=mode, modulename=modulename)) 420 421 def _group_keywords_by_length(self) -> Dict[int, List[Tuple[str, int]]]: 422 groups: Dict[int, List[Tuple[str, int]]] = {} 423 for keyword_str, keyword_type in self.callmakervisitor.keyword_cache.items(): 424 length = len(keyword_str) 425 if length in groups: 426 groups[length].append((keyword_str, keyword_type)) 427 else: 428 groups[length] = [(keyword_str, keyword_type)] 429 return groups 430 431 def _setup_keywords(self) -> None: 432 keyword_cache = self.callmakervisitor.keyword_cache 433 n_keyword_lists = ( 434 len(max(keyword_cache.keys(), key=len)) + 1 if len(keyword_cache) > 0 else 0 435 ) 436 self.print(f"static const int n_keyword_lists = {n_keyword_lists};") 437 groups = self._group_keywords_by_length() 438 self.print("static KeywordToken *reserved_keywords[] = {") 439 with self.indent(): 440 num_groups = max(groups) + 1 if groups else 1 441 for keywords_length in range(num_groups): 442 if keywords_length not in groups.keys(): 443 self.print("(KeywordToken[]) {{NULL, -1}},") 444 else: 445 self.print("(KeywordToken[]) {") 446 with self.indent(): 447 for keyword_str, keyword_type in groups[keywords_length]: 448 self.print(f'{{"{keyword_str}", {keyword_type}}},') 449 self.print("{NULL, -1},") 450 self.print("},") 451 self.print("};") 452 453 def _set_up_token_start_metadata_extraction(self) -> None: 454 self.print("if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {") 455 with self.indent(): 456 self.print("p->error_indicator = 1;") 457 self.add_return("NULL") 458 self.print("}") 459 self.print("int _start_lineno = p->tokens[_mark]->lineno;") 460 self.print("UNUSED(_start_lineno); // Only used by EXTRA macro") 461 self.print("int _start_col_offset = p->tokens[_mark]->col_offset;") 462 self.print("UNUSED(_start_col_offset); // Only used by EXTRA macro") 463 464 def _set_up_token_end_metadata_extraction(self) -> None: 465 self.print("Token *_token = _PyPegen_get_last_nonnwhitespace_token(p);") 466 self.print("if (_token == NULL) {") 467 with self.indent(): 468 self.add_return("NULL") 469 self.print("}") 470 self.print("int _end_lineno = _token->end_lineno;") 471 self.print("UNUSED(_end_lineno); // Only used by EXTRA macro") 472 self.print("int _end_col_offset = _token->end_col_offset;") 473 self.print("UNUSED(_end_col_offset); // Only used by EXTRA macro") 474 475 def _check_for_errors(self) -> None: 476 self.print("if (p->error_indicator) {") 477 with self.indent(): 478 self.add_return("NULL") 479 self.print("}") 480 481 def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None: 482 self.print("{") 483 with self.indent(): 484 self.add_level() 485 self.print(f"{result_type} _res = NULL;") 486 self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") 487 with self.indent(): 488 self.add_return("_res") 489 self.print("}") 490 self.print("int _mark = p->mark;") 491 self.print("int _resmark = p->mark;") 492 self.print("while (1) {") 493 with self.indent(): 494 self.call_with_errorcheck_return( 495 f"_PyPegen_update_memo(p, _mark, {node.name}_type, _res)", "_res" 496 ) 497 self.print("p->mark = _mark;") 498 self.print(f"void *_raw = {node.name}_raw(p);") 499 self.print("if (p->error_indicator)") 500 with self.indent(): 501 self.print("return NULL;") 502 self.print("if (_raw == NULL || p->mark <= _resmark)") 503 with self.indent(): 504 self.print("break;") 505 self.print(f"_resmark = p->mark;") 506 self.print("_res = _raw;") 507 self.print("}") 508 self.print(f"p->mark = _resmark;") 509 self.add_return("_res") 510 self.print("}") 511 self.print(f"static {result_type}") 512 self.print(f"{node.name}_raw(Parser *p)") 513 514 def _should_memoize(self, node: Rule) -> bool: 515 return node.memo and not node.left_recursive 516 517 def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> None: 518 memoize = self._should_memoize(node) 519 520 with self.indent(): 521 self.add_level() 522 self._check_for_errors() 523 self.print(f"{result_type} _res = NULL;") 524 if memoize: 525 self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") 526 with self.indent(): 527 self.add_return("_res") 528 self.print("}") 529 self.print("int _mark = p->mark;") 530 if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): 531 self._set_up_token_start_metadata_extraction() 532 self.visit( 533 rhs, is_loop=False, is_gather=node.is_gather(), rulename=node.name, 534 ) 535 if self.debug: 536 self.print(f'D(fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark));') 537 self.print("_res = NULL;") 538 self.print(" done:") 539 with self.indent(): 540 if memoize: 541 self.print(f"_PyPegen_insert_memo(p, _mark, {node.name}_type, _res);") 542 self.add_return("_res") 543 544 def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: 545 memoize = self._should_memoize(node) 546 is_repeat1 = node.name.startswith("_loop1") 547 548 with self.indent(): 549 self.add_level() 550 self._check_for_errors() 551 self.print("void *_res = NULL;") 552 if memoize: 553 self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") 554 with self.indent(): 555 self.add_return("_res") 556 self.print("}") 557 self.print("int _mark = p->mark;") 558 self.print("int _start_mark = p->mark;") 559 self.print("void **_children = PyMem_Malloc(sizeof(void *));") 560 self.out_of_memory_return(f"!_children") 561 self.print("ssize_t _children_capacity = 1;") 562 self.print("ssize_t _n = 0;") 563 if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): 564 self._set_up_token_start_metadata_extraction() 565 self.visit( 566 rhs, is_loop=True, is_gather=node.is_gather(), rulename=node.name, 567 ) 568 if is_repeat1: 569 self.print("if (_n == 0 || p->error_indicator) {") 570 with self.indent(): 571 self.print("PyMem_Free(_children);") 572 self.add_return("NULL") 573 self.print("}") 574 self.print("asdl_seq *_seq = _Py_asdl_seq_new(_n, p->arena);") 575 self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);") 576 self.print("for (int i = 0; i < _n; i++) asdl_seq_SET(_seq, i, _children[i]);") 577 self.print("PyMem_Free(_children);") 578 if node.name: 579 self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") 580 self.add_return("_seq") 581 582 def visit_Rule(self, node: Rule) -> None: 583 is_loop = node.is_loop() 584 is_gather = node.is_gather() 585 rhs = node.flatten() 586 if is_loop or is_gather: 587 result_type = "asdl_seq *" 588 elif node.type: 589 result_type = node.type 590 else: 591 result_type = "void *" 592 593 for line in str(node).splitlines(): 594 self.print(f"// {line}") 595 if node.left_recursive and node.leader: 596 self.print(f"static {result_type} {node.name}_raw(Parser *);") 597 598 self.print(f"static {result_type}") 599 self.print(f"{node.name}_rule(Parser *p)") 600 601 if node.left_recursive and node.leader: 602 self._set_up_rule_memoization(node, result_type) 603 604 self.print("{") 605 if is_loop: 606 self._handle_loop_rule_body(node, rhs) 607 else: 608 self._handle_default_rule_body(node, rhs, result_type) 609 self.print("}") 610 611 def visit_NamedItem(self, node: NamedItem) -> None: 612 call = self.callmakervisitor.generate_call(node) 613 if call.assigned_variable: 614 call.assigned_variable = self.dedupe(call.assigned_variable) 615 self.print(call) 616 617 def visit_Rhs( 618 self, node: Rhs, is_loop: bool, is_gather: bool, rulename: Optional[str] 619 ) -> None: 620 if is_loop: 621 assert len(node.alts) == 1 622 for alt in node.alts: 623 self.visit(alt, is_loop=is_loop, is_gather=is_gather, rulename=rulename) 624 625 def join_conditions(self, keyword: str, node: Any) -> None: 626 self.print(f"{keyword} (") 627 with self.indent(): 628 first = True 629 for item in node.items: 630 if first: 631 first = False 632 else: 633 self.print("&&") 634 self.visit(item) 635 self.print(")") 636 637 def emit_action(self, node: Alt, cleanup_code: Optional[str] = None) -> None: 638 self.print(f"_res = {node.action};") 639 640 self.print("if (_res == NULL && PyErr_Occurred()) {") 641 with self.indent(): 642 self.print("p->error_indicator = 1;") 643 if cleanup_code: 644 self.print(cleanup_code) 645 self.add_return("NULL") 646 self.print("}") 647 648 if self.debug: 649 self.print( 650 f'D(fprintf(stderr, "Hit with action [%d-%d]: %s\\n", _mark, p->mark, "{node}"));' 651 ) 652 653 def emit_default_action(self, is_gather: bool, node: Alt) -> None: 654 if len(self.local_variable_names) > 1: 655 if is_gather: 656 assert len(self.local_variable_names) == 2 657 self.print( 658 f"_res = _PyPegen_seq_insert_in_front(p, " 659 f"{self.local_variable_names[0]}, {self.local_variable_names[1]});" 660 ) 661 else: 662 if self.debug: 663 self.print( 664 f'D(fprintf(stderr, "Hit without action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));' 665 ) 666 self.print( 667 f"_res = _PyPegen_dummy_name(p, {', '.join(self.local_variable_names)});" 668 ) 669 else: 670 if self.debug: 671 self.print( 672 f'D(fprintf(stderr, "Hit with default action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));' 673 ) 674 self.print(f"_res = {self.local_variable_names[0]};") 675 676 def emit_dummy_action(self) -> None: 677 self.print("_res = _PyPegen_dummy_name(p);") 678 679 def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: Optional[str]) -> None: 680 self.join_conditions(keyword="if", node=node) 681 self.print("{") 682 # We have parsed successfully all the conditions for the option. 683 with self.indent(): 684 node_str = str(node).replace('"', '\\"') 685 self.print( 686 f'D(fprintf(stderr, "%*c+ {rulename}[%d-%d]: %s succeeded!\\n", p->level, \' \', _mark, p->mark, "{node_str}"));' 687 ) 688 # Prepare to emmit the rule action and do so 689 if node.action and "EXTRA" in node.action: 690 self._set_up_token_end_metadata_extraction() 691 if self.skip_actions: 692 self.emit_dummy_action() 693 elif node.action: 694 self.emit_action(node) 695 else: 696 self.emit_default_action(is_gather, node) 697 698 # As the current option has parsed correctly, do not continue with the rest. 699 self.print(f"goto done;") 700 self.print("}") 701 702 def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: Optional[str]) -> None: 703 # Condition of the main body of the alternative 704 self.join_conditions(keyword="while", node=node) 705 self.print("{") 706 # We have parsed successfully one item! 707 with self.indent(): 708 # Prepare to emit the rule action and do so 709 if node.action and "EXTRA" in node.action: 710 self._set_up_token_end_metadata_extraction() 711 if self.skip_actions: 712 self.emit_dummy_action() 713 elif node.action: 714 self.emit_action(node, cleanup_code="PyMem_Free(_children);") 715 else: 716 self.emit_default_action(is_gather, node) 717 718 # Add the result of rule to the temporary buffer of children. This buffer 719 # will populate later an asdl_seq with all elements to return. 720 self.print("if (_n == _children_capacity) {") 721 with self.indent(): 722 self.print("_children_capacity *= 2;") 723 self.print( 724 "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" 725 ) 726 self.out_of_memory_return(f"!_new_children") 727 self.print("_children = _new_children;") 728 self.print("}") 729 self.print("_children[_n++] = _res;") 730 self.print("_mark = p->mark;") 731 self.print("}") 732 733 def visit_Alt( 734 self, node: Alt, is_loop: bool, is_gather: bool, rulename: Optional[str] 735 ) -> None: 736 if len(node.items) == 1 and str(node.items[0]).startswith('invalid_'): 737 self.print(f"if (p->call_invalid_rules) {{ // {node}") 738 else: 739 self.print(f"{{ // {node}") 740 with self.indent(): 741 self._check_for_errors() 742 node_str = str(node).replace('"', '\\"') 743 self.print( 744 f'D(fprintf(stderr, "%*c> {rulename}[%d-%d]: %s\\n", p->level, \' \', _mark, p->mark, "{node_str}"));' 745 ) 746 # Prepare variable declarations for the alternative 747 vars = self.collect_vars(node) 748 for v, var_type in sorted(item for item in vars.items() if item[0] is not None): 749 if not var_type: 750 var_type = "void *" 751 else: 752 var_type += " " 753 if v == "_cut_var": 754 v += " = 0" # cut_var must be initialized 755 self.print(f"{var_type}{v};") 756 if v.startswith("_opt_var"): 757 self.print(f"UNUSED({v}); // Silence compiler warnings") 758 759 with self.local_variable_context(): 760 if is_loop: 761 self.handle_alt_loop(node, is_gather, rulename) 762 else: 763 self.handle_alt_normal(node, is_gather, rulename) 764 765 self.print("p->mark = _mark;") 766 node_str = str(node).replace('"', '\\"') 767 self.print( 768 f"D(fprintf(stderr, \"%*c%s {rulename}[%d-%d]: %s failed!\\n\", p->level, ' ',\n" 769 f' p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "{node_str}"));' 770 ) 771 if "_cut_var" in vars: 772 self.print("if (_cut_var) {") 773 with self.indent(): 774 self.add_return("NULL") 775 self.print("}") 776 self.print("}") 777 778 def collect_vars(self, node: Alt) -> Dict[Optional[str], Optional[str]]: 779 types = {} 780 with self.local_variable_context(): 781 for item in node.items: 782 name, type = self.add_var(item) 783 types[name] = type 784 return types 785 786 def add_var(self, node: NamedItem) -> Tuple[Optional[str], Optional[str]]: 787 call = self.callmakervisitor.generate_call(node.item) 788 name = node.name if node.name else call.assigned_variable 789 if name is not None: 790 name = self.dedupe(name) 791 return name, call.return_type 792