1 /* parsermodule.c
2 *
3 * Copyright 1995-1996 by Fred L. Drake, Jr. and Virginia Polytechnic
4 * Institute and State University, Blacksburg, Virginia, USA.
5 * Portions copyright 1991-1995 by Stichting Mathematisch Centrum,
6 * Amsterdam, The Netherlands. Copying is permitted under the terms
7 * associated with the main Python distribution, with the additional
8 * restriction that this additional notice be included and maintained
9 * on all distributed copies.
10 *
11 * This module serves to replace the original parser module written
12 * by Guido. The functionality is not matched precisely, but the
13 * original may be implemented on top of this. This is desirable
14 * since the source of the text to be parsed is now divorced from
15 * this interface.
16 *
17 * Unlike the prior interface, the ability to give a parse tree
18 * produced by Python code as a tuple to the compiler is enabled by
19 * this module. See the documentation for more details.
20 *
21 * I've added some annotations that help with the lint code-checking
22 * program, but they're not complete by a long shot. The real errors
23 * that lint detects are gone, but there are still warnings with
24 * Py_[X]DECREF() and Py_[X]INCREF() macros. The lint annotations
25 * look like "NOTE(...)".
26 */
27
28 #include "Python.h" /* general Python API */
29 #include "Python-ast.h" /* mod_ty */
30 #include "graminit.h" /* symbols defined in the grammar */
31 #include "node.h" /* internal parser structure */
32 #include "errcode.h" /* error codes for PyNode_*() */
33 #include "token.h" /* token definitions */
34 #include "grammar.h"
35 #include "parsetok.h"
36 /* ISTERMINAL() / ISNONTERMINAL() */
37 #include "compile.h"
38 #undef Yield
39 #include "ast.h"
40 #include "pyarena.h"
41
42 extern grammar _PyParser_Grammar; /* From graminit.c */
43
44 #ifdef lint
45 #include <note.h>
46 #else
47 #define NOTE(x)
48 #endif
49
50 /* String constants used to initialize module attributes.
51 *
52 */
53 static char parser_copyright_string[] =
54 "Copyright 1995-1996 by Virginia Polytechnic Institute & State\n\
55 University, Blacksburg, Virginia, USA, and Fred L. Drake, Jr., Reston,\n\
56 Virginia, USA. Portions copyright 1991-1995 by Stichting Mathematisch\n\
57 Centrum, Amsterdam, The Netherlands.";
58
59
60 PyDoc_STRVAR(parser_doc_string,
61 "This is an interface to Python's internal parser.");
62
63 static char parser_version_string[] = "0.5";
64
65
66 typedef PyObject* (*SeqMaker) (Py_ssize_t length);
67 typedef int (*SeqInserter) (PyObject* sequence,
68 Py_ssize_t index,
69 PyObject* element);
70
71 /* The function below is copyrighted by Stichting Mathematisch Centrum. The
72 * original copyright statement is included below, and continues to apply
73 * in full to the function immediately following. All other material is
74 * original, copyrighted by Fred L. Drake, Jr. and Virginia Polytechnic
75 * Institute and State University. Changes were made to comply with the
76 * new naming conventions. Added arguments to provide support for creating
77 * lists as well as tuples, and optionally including the line numbers.
78 */
79
80
81 static PyObject*
node2tuple(node * n,SeqMaker mkseq,SeqInserter addelem,int lineno,int col_offset)82 node2tuple(node *n, /* node to convert */
83 SeqMaker mkseq, /* create sequence */
84 SeqInserter addelem, /* func. to add elem. in seq. */
85 int lineno, /* include line numbers? */
86 int col_offset) /* include column offsets? */
87 {
88 if (n == NULL) {
89 Py_INCREF(Py_None);
90 return (Py_None);
91 }
92 if (ISNONTERMINAL(TYPE(n))) {
93 int i;
94 PyObject *v;
95 PyObject *w;
96
97 v = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl));
98 if (v == NULL)
99 return (v);
100 w = PyInt_FromLong(TYPE(n));
101 if (w == NULL) {
102 Py_DECREF(v);
103 return ((PyObject*) NULL);
104 }
105 (void) addelem(v, 0, w);
106 for (i = 0; i < NCH(n); i++) {
107 w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset);
108 if (w == NULL) {
109 Py_DECREF(v);
110 return ((PyObject*) NULL);
111 }
112 (void) addelem(v, i+1, w);
113 }
114
115 if (TYPE(n) == encoding_decl)
116 (void) addelem(v, i+1, PyString_FromString(STR(n)));
117 return (v);
118 }
119 else if (ISTERMINAL(TYPE(n))) {
120 PyObject *result = mkseq(2 + lineno + col_offset);
121 if (result != NULL) {
122 (void) addelem(result, 0, PyInt_FromLong(TYPE(n)));
123 (void) addelem(result, 1, PyString_FromString(STR(n)));
124 if (lineno == 1)
125 (void) addelem(result, 2, PyInt_FromLong(n->n_lineno));
126 if (col_offset == 1)
127 (void) addelem(result, 3, PyInt_FromLong(n->n_col_offset));
128 }
129 return (result);
130 }
131 else {
132 PyErr_SetString(PyExc_SystemError,
133 "unrecognized parse tree node type");
134 return ((PyObject*) NULL);
135 }
136 }
137 /*
138 * End of material copyrighted by Stichting Mathematisch Centrum.
139 */
140
141
142
143 /* There are two types of intermediate objects we're interested in:
144 * 'eval' and 'exec' types. These constants can be used in the st_type
145 * field of the object type to identify which any given object represents.
146 * These should probably go in an external header to allow other extensions
147 * to use them, but then, we really should be using C++ too. ;-)
148 */
149
150 #define PyST_EXPR 1
151 #define PyST_SUITE 2
152
153
154 /* These are the internal objects and definitions required to implement the
155 * ST type. Most of the internal names are more reminiscent of the 'old'
156 * naming style, but the code uses the new naming convention.
157 */
158
159 static PyObject*
160 parser_error = 0;
161
162
163 typedef struct {
164 PyObject_HEAD /* standard object header */
165 node* st_node; /* the node* returned by the parser */
166 int st_type; /* EXPR or SUITE ? */
167 PyCompilerFlags st_flags; /* Parser and compiler flags */
168 } PyST_Object;
169
170
171 static void parser_free(PyST_Object *st);
172 static PyObject* parser_sizeof(PyST_Object *, void *);
173 static int parser_compare(PyST_Object *left, PyST_Object *right);
174 static PyObject *parser_getattr(PyObject *self, char *name);
175 static PyObject* parser_compilest(PyST_Object *, PyObject *, PyObject *);
176 static PyObject* parser_isexpr(PyST_Object *, PyObject *, PyObject *);
177 static PyObject* parser_issuite(PyST_Object *, PyObject *, PyObject *);
178 static PyObject* parser_st2list(PyST_Object *, PyObject *, PyObject *);
179 static PyObject* parser_st2tuple(PyST_Object *, PyObject *, PyObject *);
180
181 #define PUBLIC_METHOD_TYPE (METH_VARARGS|METH_KEYWORDS)
182
183 static PyMethodDef
184 parser_methods[] = {
185 {"compile", (PyCFunction)parser_compilest, PUBLIC_METHOD_TYPE,
186 PyDoc_STR("Compile this ST object into a code object.")},
187 {"isexpr", (PyCFunction)parser_isexpr, PUBLIC_METHOD_TYPE,
188 PyDoc_STR("Determines if this ST object was created from an expression.")},
189 {"issuite", (PyCFunction)parser_issuite, PUBLIC_METHOD_TYPE,
190 PyDoc_STR("Determines if this ST object was created from a suite.")},
191 {"tolist", (PyCFunction)parser_st2list, PUBLIC_METHOD_TYPE,
192 PyDoc_STR("Creates a list-tree representation of this ST.")},
193 {"totuple", (PyCFunction)parser_st2tuple, PUBLIC_METHOD_TYPE,
194 PyDoc_STR("Creates a tuple-tree representation of this ST.")},
195 {"__sizeof__", (PyCFunction)parser_sizeof, METH_NOARGS,
196 PyDoc_STR("Returns size in memory, in bytes.")},
197 {NULL, NULL, 0, NULL}
198 };
199
200 static
201 PyTypeObject PyST_Type = {
202 PyVarObject_HEAD_INIT(NULL, 0)
203 "parser.st", /* tp_name */
204 (int) sizeof(PyST_Object), /* tp_basicsize */
205 0, /* tp_itemsize */
206 (destructor)parser_free, /* tp_dealloc */
207 0, /* tp_print */
208 parser_getattr, /* tp_getattr */
209 0, /* tp_setattr */
210 (cmpfunc)parser_compare, /* tp_compare */
211 0, /* tp_repr */
212 0, /* tp_as_number */
213 0, /* tp_as_sequence */
214 0, /* tp_as_mapping */
215 0, /* tp_hash */
216 0, /* tp_call */
217 0, /* tp_str */
218 0, /* tp_getattro */
219 0, /* tp_setattro */
220
221 /* Functions to access object as input/output buffer */
222 0, /* tp_as_buffer */
223
224 Py_TPFLAGS_DEFAULT, /* tp_flags */
225
226 /* __doc__ */
227 "Intermediate representation of a Python parse tree.",
228 0, /* tp_traverse */
229 0, /* tp_clear */
230 0, /* tp_richcompare */
231 0, /* tp_weaklistoffset */
232 0, /* tp_iter */
233 0, /* tp_iternext */
234 parser_methods, /* tp_methods */
235 }; /* PyST_Type */
236
237
238 static int
parser_compare_nodes(node * left,node * right)239 parser_compare_nodes(node *left, node *right)
240 {
241 int j;
242
243 if (TYPE(left) < TYPE(right))
244 return (-1);
245
246 if (TYPE(right) < TYPE(left))
247 return (1);
248
249 if (ISTERMINAL(TYPE(left)))
250 return (strcmp(STR(left), STR(right)));
251
252 if (NCH(left) < NCH(right))
253 return (-1);
254
255 if (NCH(right) < NCH(left))
256 return (1);
257
258 for (j = 0; j < NCH(left); ++j) {
259 int v = parser_compare_nodes(CHILD(left, j), CHILD(right, j));
260
261 if (v != 0)
262 return (v);
263 }
264 return (0);
265 }
266
267
268 /* int parser_compare(PyST_Object* left, PyST_Object* right)
269 *
270 * Comparison function used by the Python operators ==, !=, <, >, <=, >=
271 * This really just wraps a call to parser_compare_nodes() with some easy
272 * checks and protection code.
273 *
274 */
275 static int
parser_compare(PyST_Object * left,PyST_Object * right)276 parser_compare(PyST_Object *left, PyST_Object *right)
277 {
278 if (left == right)
279 return (0);
280
281 if ((left == 0) || (right == 0))
282 return (-1);
283
284 return (parser_compare_nodes(left->st_node, right->st_node));
285 }
286
287
288 /* parser_newstobject(node* st)
289 *
290 * Allocates a new Python object representing an ST. This is simply the
291 * 'wrapper' object that holds a node* and allows it to be passed around in
292 * Python code.
293 *
294 */
295 static PyObject*
parser_newstobject(node * st,int type)296 parser_newstobject(node *st, int type)
297 {
298 PyST_Object* o = PyObject_New(PyST_Object, &PyST_Type);
299
300 if (o != 0) {
301 o->st_node = st;
302 o->st_type = type;
303 o->st_flags.cf_flags = 0;
304 }
305 else {
306 PyNode_Free(st);
307 }
308 return ((PyObject*)o);
309 }
310
311
312 /* void parser_free(PyST_Object* st)
313 *
314 * This is called by a del statement that reduces the reference count to 0.
315 *
316 */
317 static void
parser_free(PyST_Object * st)318 parser_free(PyST_Object *st)
319 {
320 PyNode_Free(st->st_node);
321 PyObject_Del(st);
322 }
323
324
325 /* parser_st2tuple(PyObject* self, PyObject* args, PyObject* kw)
326 *
327 * This provides conversion from a node* to a tuple object that can be
328 * returned to the Python-level caller. The ST object is not modified.
329 *
330 */
331 static PyObject*
parser_st2tuple(PyST_Object * self,PyObject * args,PyObject * kw)332 parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw)
333 {
334 PyObject *line_option = 0;
335 PyObject *col_option = 0;
336 PyObject *res = 0;
337 int ok;
338
339 static char *keywords[] = {"ast", "line_info", "col_info", NULL};
340
341 if (self == NULL) {
342 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2tuple", keywords,
343 &PyST_Type, &self, &line_option,
344 &col_option);
345 }
346 else
347 ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:totuple", &keywords[1],
348 &line_option, &col_option);
349 if (ok != 0) {
350 int lineno = 0;
351 int col_offset = 0;
352 if (line_option != NULL) {
353 lineno = PyObject_IsTrue(line_option);
354 if (lineno < 0)
355 return NULL;
356 }
357 if (col_option != NULL) {
358 col_offset = PyObject_IsTrue(col_option);
359 if (col_offset < 0)
360 return NULL;
361 }
362 /*
363 * Convert ST into a tuple representation. Use Guido's function,
364 * since it's known to work already.
365 */
366 res = node2tuple(((PyST_Object*)self)->st_node,
367 PyTuple_New, PyTuple_SetItem, lineno, col_offset);
368 }
369 return (res);
370 }
371
372 static PyObject*
parser_ast2tuple(PyST_Object * self,PyObject * args,PyObject * kw)373 parser_ast2tuple(PyST_Object *self, PyObject *args, PyObject *kw)
374 {
375 if (PyErr_WarnPy3k("ast2tuple is removed in 3.x; use st2tuple", 1) < 0)
376 return NULL;
377 return parser_st2tuple(self, args, kw);
378 }
379
380
381 /* parser_st2list(PyObject* self, PyObject* args, PyObject* kw)
382 *
383 * This provides conversion from a node* to a list object that can be
384 * returned to the Python-level caller. The ST object is not modified.
385 *
386 */
387 static PyObject*
parser_st2list(PyST_Object * self,PyObject * args,PyObject * kw)388 parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw)
389 {
390 PyObject *line_option = 0;
391 PyObject *col_option = 0;
392 PyObject *res = 0;
393 int ok;
394
395 static char *keywords[] = {"ast", "line_info", "col_info", NULL};
396
397 if (self == NULL)
398 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2list", keywords,
399 &PyST_Type, &self, &line_option,
400 &col_option);
401 else
402 ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:tolist", &keywords[1],
403 &line_option, &col_option);
404 if (ok) {
405 int lineno = 0;
406 int col_offset = 0;
407 if (line_option != 0) {
408 lineno = PyObject_IsTrue(line_option);
409 if (lineno < 0)
410 return NULL;
411 }
412 if (col_option != 0) {
413 col_offset = PyObject_IsTrue(col_option);
414 if (col_offset < 0)
415 return NULL;
416 }
417 /*
418 * Convert ST into a tuple representation. Use Guido's function,
419 * since it's known to work already.
420 */
421 res = node2tuple(self->st_node,
422 PyList_New, PyList_SetItem, lineno, col_offset);
423 }
424 return (res);
425 }
426
427 static PyObject*
parser_ast2list(PyST_Object * self,PyObject * args,PyObject * kw)428 parser_ast2list(PyST_Object *self, PyObject *args, PyObject *kw)
429 {
430 if (PyErr_WarnPy3k("ast2list is removed in 3.x; use st2list", 1) < 0)
431 return NULL;
432 return parser_st2list(self, args, kw);
433 }
434
435
436 /* parser_compilest(PyObject* self, PyObject* args)
437 *
438 * This function creates code objects from the parse tree represented by
439 * the passed-in data object. An optional file name is passed in as well.
440 *
441 */
442 static PyObject*
parser_compilest(PyST_Object * self,PyObject * args,PyObject * kw)443 parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw)
444 {
445 PyObject* res = 0;
446 PyArena* arena;
447 mod_ty mod;
448 char* str = "<syntax-tree>";
449 int ok;
450
451 static char *keywords[] = {"ast", "filename", NULL};
452
453 if (self == NULL)
454 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|s:compilest", keywords,
455 &PyST_Type, &self, &str);
456 else
457 ok = PyArg_ParseTupleAndKeywords(args, kw, "|s:compile", &keywords[1],
458 &str);
459
460 if (ok) {
461 arena = PyArena_New();
462 if (arena) {
463 mod = PyAST_FromNode(self->st_node, &(self->st_flags), str, arena);
464 if (mod) {
465 res = (PyObject *)PyAST_Compile(mod, str, &(self->st_flags), arena);
466 }
467 PyArena_Free(arena);
468 }
469 }
470
471 return (res);
472 }
473
474 static PyObject*
parser_compileast(PyST_Object * self,PyObject * args,PyObject * kw)475 parser_compileast(PyST_Object *self, PyObject *args, PyObject *kw)
476 {
477 if (PyErr_WarnPy3k("compileast is removed in 3.x; use compilest", 1) < 0)
478 return NULL;
479 return parser_compilest(self, args, kw);
480 }
481
482
483 /* PyObject* parser_isexpr(PyObject* self, PyObject* args)
484 * PyObject* parser_issuite(PyObject* self, PyObject* args)
485 *
486 * Checks the passed-in ST object to determine if it is an expression or
487 * a statement suite, respectively. The return is a Python truth value.
488 *
489 */
490 static PyObject*
parser_isexpr(PyST_Object * self,PyObject * args,PyObject * kw)491 parser_isexpr(PyST_Object *self, PyObject *args, PyObject *kw)
492 {
493 PyObject* res = 0;
494 int ok;
495
496 static char *keywords[] = {"ast", NULL};
497
498 if (self == NULL)
499 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:isexpr", keywords,
500 &PyST_Type, &self);
501 else
502 ok = PyArg_ParseTupleAndKeywords(args, kw, ":isexpr", &keywords[1]);
503
504 if (ok) {
505 /* Check to see if the ST represents an expression or not. */
506 res = (self->st_type == PyST_EXPR) ? Py_True : Py_False;
507 Py_INCREF(res);
508 }
509 return (res);
510 }
511
512
513 static PyObject*
parser_issuite(PyST_Object * self,PyObject * args,PyObject * kw)514 parser_issuite(PyST_Object *self, PyObject *args, PyObject *kw)
515 {
516 PyObject* res = 0;
517 int ok;
518
519 static char *keywords[] = {"ast", NULL};
520
521 if (self == NULL)
522 ok = PyArg_ParseTupleAndKeywords(args, kw, "O!:issuite", keywords,
523 &PyST_Type, &self);
524 else
525 ok = PyArg_ParseTupleAndKeywords(args, kw, ":issuite", &keywords[1]);
526
527 if (ok) {
528 /* Check to see if the ST represents an expression or not. */
529 res = (self->st_type == PyST_EXPR) ? Py_False : Py_True;
530 Py_INCREF(res);
531 }
532 return (res);
533 }
534
535
536 static PyObject*
parser_getattr(PyObject * self,char * name)537 parser_getattr(PyObject *self, char *name)
538 {
539 return (Py_FindMethod(parser_methods, self, name));
540 }
541
542
543 /* err_string(char* message)
544 *
545 * Sets the error string for an exception of type ParserError.
546 *
547 */
548 static void
err_string(char * message)549 err_string(char *message)
550 {
551 PyErr_SetString(parser_error, message);
552 }
553
554
555 /* PyObject* parser_do_parse(PyObject* args, int type)
556 *
557 * Internal function to actually execute the parse and return the result if
558 * successful or set an exception if not.
559 *
560 */
561 static PyObject*
parser_do_parse(PyObject * args,PyObject * kw,char * argspec,int type)562 parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type)
563 {
564 char* string = 0;
565 PyObject* res = 0;
566 int flags = 0;
567 perrdetail err;
568
569 static char *keywords[] = {"source", NULL};
570
571 if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) {
572 node* n = PyParser_ParseStringFlagsFilenameEx(string, NULL,
573 &_PyParser_Grammar,
574 (type == PyST_EXPR)
575 ? eval_input : file_input,
576 &err, &flags);
577
578 if (n) {
579 res = parser_newstobject(n, type);
580 if (res)
581 ((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK;
582 }
583 else
584 PyParser_SetError(&err);
585 }
586 return (res);
587 }
588
589
590 /* PyObject* parser_expr(PyObject* self, PyObject* args)
591 * PyObject* parser_suite(PyObject* self, PyObject* args)
592 *
593 * External interfaces to the parser itself. Which is called determines if
594 * the parser attempts to recognize an expression ('eval' form) or statement
595 * suite ('exec' form). The real work is done by parser_do_parse() above.
596 *
597 */
598 static PyObject*
parser_expr(PyST_Object * self,PyObject * args,PyObject * kw)599 parser_expr(PyST_Object *self, PyObject *args, PyObject *kw)
600 {
601 NOTE(ARGUNUSED(self))
602 return (parser_do_parse(args, kw, "s:expr", PyST_EXPR));
603 }
604
605
606 static PyObject*
parser_suite(PyST_Object * self,PyObject * args,PyObject * kw)607 parser_suite(PyST_Object *self, PyObject *args, PyObject *kw)
608 {
609 NOTE(ARGUNUSED(self))
610 return (parser_do_parse(args, kw, "s:suite", PyST_SUITE));
611 }
612
613
614
615 /* This is the messy part of the code. Conversion from a tuple to an ST
616 * object requires that the input tuple be valid without having to rely on
617 * catching an exception from the compiler. This is done to allow the
618 * compiler itself to remain fast, since most of its input will come from
619 * the parser directly, and therefore be known to be syntactically correct.
620 * This validation is done to ensure that we don't core dump the compile
621 * phase, returning an exception instead.
622 *
623 * Two aspects can be broken out in this code: creating a node tree from
624 * the tuple passed in, and verifying that it is indeed valid. It may be
625 * advantageous to expand the number of ST types to include funcdefs and
626 * lambdadefs to take advantage of the optimizer, recognizing those STs
627 * here. They are not necessary, and not quite as useful in a raw form.
628 * For now, let's get expressions and suites working reliably.
629 */
630
631
632 static node* build_node_tree(PyObject *tuple);
633 static int validate_expr_tree(node *tree);
634 static int validate_file_input(node *tree);
635 static int validate_encoding_decl(node *tree);
636
637 /* PyObject* parser_tuple2st(PyObject* self, PyObject* args)
638 *
639 * This is the public function, called from the Python code. It receives a
640 * single tuple object from the caller, and creates an ST object if the
641 * tuple can be validated. It does this by checking the first code of the
642 * tuple, and, if acceptable, builds the internal representation. If this
643 * step succeeds, the internal representation is validated as fully as
644 * possible with the various validate_*() routines defined below.
645 *
646 * This function must be changed if support is to be added for PyST_FRAGMENT
647 * ST objects.
648 *
649 */
650 static PyObject*
parser_tuple2st(PyST_Object * self,PyObject * args,PyObject * kw)651 parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw)
652 {
653 NOTE(ARGUNUSED(self))
654 PyObject *st = 0;
655 PyObject *tuple;
656 node *tree;
657
658 static char *keywords[] = {"sequence", NULL};
659
660 if (!PyArg_ParseTupleAndKeywords(args, kw, "O:sequence2st", keywords,
661 &tuple))
662 return (0);
663 if (!PySequence_Check(tuple)) {
664 PyErr_SetString(PyExc_ValueError,
665 "sequence2st() requires a single sequence argument");
666 return (0);
667 }
668 /*
669 * Convert the tree to the internal form before checking it.
670 */
671 tree = build_node_tree(tuple);
672 if (tree != 0) {
673 int start_sym = TYPE(tree);
674 if (start_sym == eval_input) {
675 /* Might be an eval form. */
676 if (validate_expr_tree(tree))
677 st = parser_newstobject(tree, PyST_EXPR);
678 else
679 PyNode_Free(tree);
680 }
681 else if (start_sym == file_input) {
682 /* This looks like an exec form so far. */
683 if (validate_file_input(tree))
684 st = parser_newstobject(tree, PyST_SUITE);
685 else
686 PyNode_Free(tree);
687 }
688 else if (start_sym == encoding_decl) {
689 /* This looks like an encoding_decl so far. */
690 if (validate_encoding_decl(tree))
691 st = parser_newstobject(tree, PyST_SUITE);
692 else
693 PyNode_Free(tree);
694 }
695 else {
696 /* This is a fragment, at best. */
697 PyNode_Free(tree);
698 err_string("parse tree does not use a valid start symbol");
699 }
700 }
701 /* Make sure we raise an exception on all errors. We should never
702 * get this, but we'd do well to be sure something is done.
703 */
704 if (st == NULL && !PyErr_Occurred())
705 err_string("unspecified ST error occurred");
706
707 return st;
708 }
709
710 static PyObject*
parser_tuple2ast(PyST_Object * self,PyObject * args,PyObject * kw)711 parser_tuple2ast(PyST_Object *self, PyObject *args, PyObject *kw)
712 {
713 if (PyErr_WarnPy3k("tuple2ast is removed in 3.x; use tuple2st", 1) < 0)
714 return NULL;
715 return parser_tuple2st(self, args, kw);
716 }
717
718 static PyObject *
parser_sizeof(PyST_Object * st,void * unused)719 parser_sizeof(PyST_Object *st, void *unused)
720 {
721 Py_ssize_t res;
722
723 res = _PyObject_SIZE(Py_TYPE(st)) + _PyNode_SizeOf(st->st_node);
724 return PyLong_FromSsize_t(res);
725 }
726
727
728 /* node* build_node_children()
729 *
730 * Iterate across the children of the current non-terminal node and build
731 * their structures. If successful, return the root of this portion of
732 * the tree, otherwise, 0. Any required exception will be specified already,
733 * and no memory will have been deallocated.
734 *
735 */
736 static node*
build_node_children(PyObject * tuple,node * root,int * line_num)737 build_node_children(PyObject *tuple, node *root, int *line_num)
738 {
739 Py_ssize_t len = PyObject_Size(tuple);
740 Py_ssize_t i;
741 int err;
742
743 for (i = 1; i < len; ++i) {
744 /* elem must always be a sequence, however simple */
745 PyObject* elem = PySequence_GetItem(tuple, i);
746 int ok = elem != NULL;
747 long type = 0;
748 char *strn = 0;
749
750 if (ok)
751 ok = PySequence_Check(elem);
752 if (ok) {
753 PyObject *temp = PySequence_GetItem(elem, 0);
754 if (temp == NULL)
755 ok = 0;
756 else {
757 ok = PyInt_Check(temp);
758 if (ok)
759 type = PyInt_AS_LONG(temp);
760 Py_DECREF(temp);
761 }
762 }
763 if (!ok) {
764 PyObject *err = Py_BuildValue("os", elem,
765 "Illegal node construct.");
766 PyErr_SetObject(parser_error, err);
767 Py_XDECREF(err);
768 Py_XDECREF(elem);
769 return (0);
770 }
771 if (ISTERMINAL(type)) {
772 Py_ssize_t len = PyObject_Size(elem);
773 PyObject *temp;
774
775 if ((len != 2) && (len != 3)) {
776 err_string("terminal nodes must have 2 or 3 entries");
777 return 0;
778 }
779 temp = PySequence_GetItem(elem, 1);
780 if (temp == NULL)
781 return 0;
782 if (!PyString_Check(temp)) {
783 PyErr_Format(parser_error,
784 "second item in terminal node must be a string,"
785 " found %s",
786 Py_TYPE(temp)->tp_name);
787 Py_DECREF(temp);
788 return 0;
789 }
790 if (len == 3) {
791 PyObject *o = PySequence_GetItem(elem, 2);
792 if (o != NULL) {
793 if (PyInt_Check(o))
794 *line_num = PyInt_AS_LONG(o);
795 else {
796 PyErr_Format(parser_error,
797 "third item in terminal node must be an"
798 " integer, found %s",
799 Py_TYPE(temp)->tp_name);
800 Py_DECREF(o);
801 Py_DECREF(temp);
802 return 0;
803 }
804 Py_DECREF(o);
805 }
806 }
807 len = PyString_GET_SIZE(temp) + 1;
808 strn = (char *)PyObject_MALLOC(len);
809 if (strn != NULL)
810 (void) memcpy(strn, PyString_AS_STRING(temp), len);
811 Py_DECREF(temp);
812 }
813 else if (!ISNONTERMINAL(type)) {
814 /*
815 * It has to be one or the other; this is an error.
816 * Raise an exception.
817 */
818 PyObject *err = Py_BuildValue("os", elem, "unknown node type.");
819 PyErr_SetObject(parser_error, err);
820 Py_XDECREF(err);
821 Py_XDECREF(elem);
822 return (0);
823 }
824 err = PyNode_AddChild(root, type, strn, *line_num, 0);
825 if (err == E_NOMEM) {
826 PyObject_FREE(strn);
827 return (node *) PyErr_NoMemory();
828 }
829 if (err == E_OVERFLOW) {
830 PyObject_FREE(strn);
831 PyErr_SetString(PyExc_ValueError,
832 "unsupported number of child nodes");
833 return NULL;
834 }
835
836 if (ISNONTERMINAL(type)) {
837 node* new_child = CHILD(root, i - 1);
838
839 if (new_child != build_node_children(elem, new_child, line_num)) {
840 Py_XDECREF(elem);
841 return (0);
842 }
843 }
844 else if (type == NEWLINE) { /* It's true: we increment the */
845 ++(*line_num); /* line number *after* the newline! */
846 }
847 Py_XDECREF(elem);
848 }
849 return root;
850 }
851
852
853 static node*
build_node_tree(PyObject * tuple)854 build_node_tree(PyObject *tuple)
855 {
856 node* res = 0;
857 PyObject *temp = PySequence_GetItem(tuple, 0);
858 long num = -1;
859
860 if (temp != NULL)
861 num = PyInt_AsLong(temp);
862 Py_XDECREF(temp);
863 if (ISTERMINAL(num)) {
864 /*
865 * The tuple is simple, but it doesn't start with a start symbol.
866 * Raise an exception now and be done with it.
867 */
868 tuple = Py_BuildValue("os", tuple,
869 "Illegal syntax-tree; cannot start with terminal symbol.");
870 PyErr_SetObject(parser_error, tuple);
871 Py_XDECREF(tuple);
872 }
873 else if (ISNONTERMINAL(num)) {
874 /*
875 * Not efficient, but that can be handled later.
876 */
877 int line_num = 0;
878 PyObject *encoding = NULL;
879
880 if (num == encoding_decl) {
881 encoding = PySequence_GetItem(tuple, 2);
882 /* tuple isn't borrowed anymore here, need to DECREF */
883 tuple = PySequence_GetSlice(tuple, 0, 2);
884 }
885 res = PyNode_New(num);
886 if (res != NULL) {
887 if (res != build_node_children(tuple, res, &line_num)) {
888 PyNode_Free(res);
889 res = NULL;
890 }
891 if (res && encoding) {
892 Py_ssize_t len;
893 len = PyString_GET_SIZE(encoding) + 1;
894 res->n_str = (char *)PyObject_MALLOC(len);
895 if (res->n_str != NULL)
896 (void) memcpy(res->n_str, PyString_AS_STRING(encoding), len);
897 Py_DECREF(encoding);
898 Py_DECREF(tuple);
899 }
900 }
901 }
902 else {
903 /* The tuple is illegal -- if the number is neither TERMINAL nor
904 * NONTERMINAL, we can't use it. Not sure the implementation
905 * allows this condition, but the API doesn't preclude it.
906 */
907 PyObject *err = Py_BuildValue("os", tuple,
908 "Illegal component tuple.");
909 PyErr_SetObject(parser_error, err);
910 Py_XDECREF(err);
911 }
912
913 return (res);
914 }
915
916
917 /*
918 * Validation routines used within the validation section:
919 */
920 static int validate_terminal(node *terminal, int type, char *string);
921
922 #define validate_ampersand(ch) validate_terminal(ch, AMPER, "&")
923 #define validate_circumflex(ch) validate_terminal(ch, CIRCUMFLEX, "^")
924 #define validate_colon(ch) validate_terminal(ch, COLON, ":")
925 #define validate_comma(ch) validate_terminal(ch, COMMA, ",")
926 #define validate_dedent(ch) validate_terminal(ch, DEDENT, "")
927 #define validate_equal(ch) validate_terminal(ch, EQUAL, "=")
928 #define validate_indent(ch) validate_terminal(ch, INDENT, (char*)NULL)
929 #define validate_lparen(ch) validate_terminal(ch, LPAR, "(")
930 #define validate_newline(ch) validate_terminal(ch, NEWLINE, (char*)NULL)
931 #define validate_rparen(ch) validate_terminal(ch, RPAR, ")")
932 #define validate_semi(ch) validate_terminal(ch, SEMI, ";")
933 #define validate_star(ch) validate_terminal(ch, STAR, "*")
934 #define validate_vbar(ch) validate_terminal(ch, VBAR, "|")
935 #define validate_doublestar(ch) validate_terminal(ch, DOUBLESTAR, "**")
936 #define validate_dot(ch) validate_terminal(ch, DOT, ".")
937 #define validate_at(ch) validate_terminal(ch, AT, "@")
938 #define validate_name(ch, str) validate_terminal(ch, NAME, str)
939
940 #define VALIDATER(n) static int validate_##n(node *tree)
941
942 VALIDATER(node); VALIDATER(small_stmt);
943 VALIDATER(class); VALIDATER(node);
944 VALIDATER(parameters); VALIDATER(suite);
945 VALIDATER(testlist); VALIDATER(varargslist);
946 VALIDATER(fpdef); VALIDATER(fplist);
947 VALIDATER(stmt); VALIDATER(simple_stmt);
948 VALIDATER(expr_stmt); VALIDATER(power);
949 VALIDATER(print_stmt); VALIDATER(del_stmt);
950 VALIDATER(return_stmt); VALIDATER(list_iter);
951 VALIDATER(raise_stmt); VALIDATER(import_stmt);
952 VALIDATER(import_name); VALIDATER(import_from);
953 VALIDATER(global_stmt); VALIDATER(list_if);
954 VALIDATER(assert_stmt); VALIDATER(list_for);
955 VALIDATER(exec_stmt); VALIDATER(compound_stmt);
956 VALIDATER(while); VALIDATER(for);
957 VALIDATER(try); VALIDATER(except_clause);
958 VALIDATER(test); VALIDATER(and_test);
959 VALIDATER(not_test); VALIDATER(comparison);
960 VALIDATER(comp_op); VALIDATER(expr);
961 VALIDATER(xor_expr); VALIDATER(and_expr);
962 VALIDATER(shift_expr); VALIDATER(arith_expr);
963 VALIDATER(term); VALIDATER(factor);
964 VALIDATER(atom); VALIDATER(lambdef);
965 VALIDATER(trailer); VALIDATER(subscript);
966 VALIDATER(subscriptlist); VALIDATER(sliceop);
967 VALIDATER(exprlist); VALIDATER(dictorsetmaker);
968 VALIDATER(arglist); VALIDATER(argument);
969 VALIDATER(listmaker); VALIDATER(yield_stmt);
970 VALIDATER(testlist1); VALIDATER(comp_for);
971 VALIDATER(comp_iter); VALIDATER(comp_if);
972 VALIDATER(testlist_comp); VALIDATER(yield_expr);
973 VALIDATER(yield_or_testlist); VALIDATER(or_test);
974 VALIDATER(old_test); VALIDATER(old_lambdef);
975
976 #undef VALIDATER
977
978 #define is_even(n) (((n) & 1) == 0)
979 #define is_odd(n) (((n) & 1) == 1)
980
981
982 static int
validate_ntype(node * n,int t)983 validate_ntype(node *n, int t)
984 {
985 if (TYPE(n) != t) {
986 PyErr_Format(parser_error, "Expected node type %d, got %d.",
987 t, TYPE(n));
988 return 0;
989 }
990 return 1;
991 }
992
993
994 /* Verifies that the number of child nodes is exactly 'num', raising
995 * an exception if it isn't. The exception message does not indicate
996 * the exact number of nodes, allowing this to be used to raise the
997 * "right" exception when the wrong number of nodes is present in a
998 * specific variant of a statement's syntax. This is commonly used
999 * in that fashion.
1000 */
1001 static int
validate_numnodes(node * n,int num,const char * const name)1002 validate_numnodes(node *n, int num, const char *const name)
1003 {
1004 if (NCH(n) != num) {
1005 PyErr_Format(parser_error,
1006 "Illegal number of children for %s node.", name);
1007 return 0;
1008 }
1009 return 1;
1010 }
1011
1012
1013 static int
validate_terminal(node * terminal,int type,char * string)1014 validate_terminal(node *terminal, int type, char *string)
1015 {
1016 int res = (validate_ntype(terminal, type)
1017 && ((string == 0) || (strcmp(string, STR(terminal)) == 0)));
1018
1019 if (!res && !PyErr_Occurred()) {
1020 PyErr_Format(parser_error,
1021 "Illegal terminal: expected \"%s\"", string);
1022 }
1023 return (res);
1024 }
1025
1026
1027 /* X (',' X) [',']
1028 */
1029 static int
validate_repeating_list(node * tree,int ntype,int (* vfunc)(node *),const char * const name)1030 validate_repeating_list(node *tree, int ntype, int (*vfunc)(node *),
1031 const char *const name)
1032 {
1033 int nch = NCH(tree);
1034 int res = (nch && validate_ntype(tree, ntype)
1035 && vfunc(CHILD(tree, 0)));
1036
1037 if (!res && !PyErr_Occurred())
1038 (void) validate_numnodes(tree, 1, name);
1039 else {
1040 if (is_even(nch))
1041 res = validate_comma(CHILD(tree, --nch));
1042 if (res && nch > 1) {
1043 int pos = 1;
1044 for ( ; res && pos < nch; pos += 2)
1045 res = (validate_comma(CHILD(tree, pos))
1046 && vfunc(CHILD(tree, pos + 1)));
1047 }
1048 }
1049 return (res);
1050 }
1051
1052
1053 /* validate_class()
1054 *
1055 * classdef:
1056 * 'class' NAME ['(' testlist ')'] ':' suite
1057 */
1058 static int
validate_class(node * tree)1059 validate_class(node *tree)
1060 {
1061 int nch = NCH(tree);
1062 int res = (validate_ntype(tree, classdef) &&
1063 ((nch == 4) || (nch == 6) || (nch == 7)));
1064
1065 if (res) {
1066 res = (validate_name(CHILD(tree, 0), "class")
1067 && validate_ntype(CHILD(tree, 1), NAME)
1068 && validate_colon(CHILD(tree, nch - 2))
1069 && validate_suite(CHILD(tree, nch - 1)));
1070 }
1071 else {
1072 (void) validate_numnodes(tree, 4, "class");
1073 }
1074
1075 if (res) {
1076 if (nch == 7) {
1077 res = ((validate_lparen(CHILD(tree, 2)) &&
1078 validate_testlist(CHILD(tree, 3)) &&
1079 validate_rparen(CHILD(tree, 4))));
1080 }
1081 else if (nch == 6) {
1082 res = (validate_lparen(CHILD(tree,2)) &&
1083 validate_rparen(CHILD(tree,3)));
1084 }
1085 }
1086 return (res);
1087 }
1088
1089
1090 /* if_stmt:
1091 * 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
1092 */
1093 static int
validate_if(node * tree)1094 validate_if(node *tree)
1095 {
1096 int nch = NCH(tree);
1097 int res = (validate_ntype(tree, if_stmt)
1098 && (nch >= 4)
1099 && validate_name(CHILD(tree, 0), "if")
1100 && validate_test(CHILD(tree, 1))
1101 && validate_colon(CHILD(tree, 2))
1102 && validate_suite(CHILD(tree, 3)));
1103
1104 if (res && ((nch % 4) == 3)) {
1105 /* ... 'else' ':' suite */
1106 res = (validate_name(CHILD(tree, nch - 3), "else")
1107 && validate_colon(CHILD(tree, nch - 2))
1108 && validate_suite(CHILD(tree, nch - 1)));
1109 nch -= 3;
1110 }
1111 else if (!res && !PyErr_Occurred())
1112 (void) validate_numnodes(tree, 4, "if");
1113 if ((nch % 4) != 0)
1114 /* Will catch the case for nch < 4 */
1115 res = validate_numnodes(tree, 0, "if");
1116 else if (res && (nch > 4)) {
1117 /* ... ('elif' test ':' suite)+ ... */
1118 int j = 4;
1119 while ((j < nch) && res) {
1120 res = (validate_name(CHILD(tree, j), "elif")
1121 && validate_colon(CHILD(tree, j + 2))
1122 && validate_test(CHILD(tree, j + 1))
1123 && validate_suite(CHILD(tree, j + 3)));
1124 j += 4;
1125 }
1126 }
1127 return (res);
1128 }
1129
1130
1131 /* parameters:
1132 * '(' [varargslist] ')'
1133 *
1134 */
1135 static int
validate_parameters(node * tree)1136 validate_parameters(node *tree)
1137 {
1138 int nch = NCH(tree);
1139 int res = validate_ntype(tree, parameters) && ((nch == 2) || (nch == 3));
1140
1141 if (res) {
1142 res = (validate_lparen(CHILD(tree, 0))
1143 && validate_rparen(CHILD(tree, nch - 1)));
1144 if (res && (nch == 3))
1145 res = validate_varargslist(CHILD(tree, 1));
1146 }
1147 else {
1148 (void) validate_numnodes(tree, 2, "parameters");
1149 }
1150 return (res);
1151 }
1152
1153
1154 /* validate_suite()
1155 *
1156 * suite:
1157 * simple_stmt
1158 * | NEWLINE INDENT stmt+ DEDENT
1159 */
1160 static int
validate_suite(node * tree)1161 validate_suite(node *tree)
1162 {
1163 int nch = NCH(tree);
1164 int res = (validate_ntype(tree, suite) && ((nch == 1) || (nch >= 4)));
1165
1166 if (res && (nch == 1))
1167 res = validate_simple_stmt(CHILD(tree, 0));
1168 else if (res) {
1169 /* NEWLINE INDENT stmt+ DEDENT */
1170 res = (validate_newline(CHILD(tree, 0))
1171 && validate_indent(CHILD(tree, 1))
1172 && validate_stmt(CHILD(tree, 2))
1173 && validate_dedent(CHILD(tree, nch - 1)));
1174
1175 if (res && (nch > 4)) {
1176 int i = 3;
1177 --nch; /* forget the DEDENT */
1178 for ( ; res && (i < nch); ++i)
1179 res = validate_stmt(CHILD(tree, i));
1180 }
1181 else if (nch < 4)
1182 res = validate_numnodes(tree, 4, "suite");
1183 }
1184 return (res);
1185 }
1186
1187
1188 static int
validate_testlist(node * tree)1189 validate_testlist(node *tree)
1190 {
1191 return (validate_repeating_list(tree, testlist,
1192 validate_test, "testlist"));
1193 }
1194
1195
1196 static int
validate_testlist1(node * tree)1197 validate_testlist1(node *tree)
1198 {
1199 return (validate_repeating_list(tree, testlist1,
1200 validate_test, "testlist1"));
1201 }
1202
1203
1204 static int
validate_testlist_safe(node * tree)1205 validate_testlist_safe(node *tree)
1206 {
1207 return (validate_repeating_list(tree, testlist_safe,
1208 validate_old_test, "testlist_safe"));
1209 }
1210
1211
1212 /* '*' NAME [',' '**' NAME] | '**' NAME
1213 */
1214 static int
validate_varargslist_trailer(node * tree,int start)1215 validate_varargslist_trailer(node *tree, int start)
1216 {
1217 int nch = NCH(tree);
1218 int res = 0;
1219 int sym;
1220
1221 if (nch <= start) {
1222 err_string("expected variable argument trailer for varargslist");
1223 return 0;
1224 }
1225 sym = TYPE(CHILD(tree, start));
1226 if (sym == STAR) {
1227 /*
1228 * ('*' NAME [',' '**' NAME]
1229 */
1230 if (nch-start == 2)
1231 res = validate_name(CHILD(tree, start+1), NULL);
1232 else if (nch-start == 5)
1233 res = (validate_name(CHILD(tree, start+1), NULL)
1234 && validate_comma(CHILD(tree, start+2))
1235 && validate_doublestar(CHILD(tree, start+3))
1236 && validate_name(CHILD(tree, start+4), NULL));
1237 }
1238 else if (sym == DOUBLESTAR) {
1239 /*
1240 * '**' NAME
1241 */
1242 if (nch-start == 2)
1243 res = validate_name(CHILD(tree, start+1), NULL);
1244 }
1245 if (!res)
1246 err_string("illegal variable argument trailer for varargslist");
1247 return res;
1248 }
1249
1250
1251 /* validate_varargslist()
1252 *
1253 * varargslist:
1254 * (fpdef ['=' test] ',')*
1255 * ('*' NAME [',' '**' NAME]
1256 * | '**' NAME)
1257 * | fpdef ['=' test] (',' fpdef ['=' test])* [',']
1258 *
1259 */
1260 static int
validate_varargslist(node * tree)1261 validate_varargslist(node *tree)
1262 {
1263 int nch = NCH(tree);
1264 int res = validate_ntype(tree, varargslist) && (nch != 0);
1265 int sym;
1266
1267 if (!res)
1268 return 0;
1269 if (nch < 1) {
1270 err_string("varargslist missing child nodes");
1271 return 0;
1272 }
1273 sym = TYPE(CHILD(tree, 0));
1274 if (sym == STAR || sym == DOUBLESTAR)
1275 /* whole thing matches:
1276 * '*' NAME [',' '**' NAME] | '**' NAME
1277 */
1278 res = validate_varargslist_trailer(tree, 0);
1279 else if (sym == fpdef) {
1280 int i = 0;
1281
1282 sym = TYPE(CHILD(tree, nch-1));
1283 if (sym == NAME) {
1284 /*
1285 * (fpdef ['=' test] ',')+
1286 * ('*' NAME [',' '**' NAME]
1287 * | '**' NAME)
1288 */
1289 /* skip over (fpdef ['=' test] ',')+ */
1290 while (res && (i+2 <= nch)) {
1291 res = validate_fpdef(CHILD(tree, i));
1292 ++i;
1293 if (res && TYPE(CHILD(tree, i)) == EQUAL && (i+2 <= nch)) {
1294 res = (validate_equal(CHILD(tree, i))
1295 && validate_test(CHILD(tree, i+1)));
1296 if (res)
1297 i += 2;
1298 }
1299 if (res && i < nch) {
1300 res = validate_comma(CHILD(tree, i));
1301 ++i;
1302 if (res && i < nch
1303 && (TYPE(CHILD(tree, i)) == DOUBLESTAR
1304 || TYPE(CHILD(tree, i)) == STAR))
1305 break;
1306 }
1307 }
1308 /* ... '*' NAME [',' '**' NAME] | '**' NAME
1309 * i --^^^
1310 */
1311 if (res)
1312 res = validate_varargslist_trailer(tree, i);
1313 }
1314 else {
1315 /*
1316 * fpdef ['=' test] (',' fpdef ['=' test])* [',']
1317 */
1318 /* strip trailing comma node */
1319 if (sym == COMMA) {
1320 res = validate_comma(CHILD(tree, nch-1));
1321 if (!res)
1322 return 0;
1323 --nch;
1324 }
1325 /*
1326 * fpdef ['=' test] (',' fpdef ['=' test])*
1327 */
1328 res = validate_fpdef(CHILD(tree, 0));
1329 ++i;
1330 if (res && (i+2 <= nch) && TYPE(CHILD(tree, i)) == EQUAL) {
1331 res = (validate_equal(CHILD(tree, i))
1332 && validate_test(CHILD(tree, i+1)));
1333 i += 2;
1334 }
1335 /*
1336 * ... (',' fpdef ['=' test])*
1337 * i ---^^^
1338 */
1339 while (res && (nch - i) >= 2) {
1340 res = (validate_comma(CHILD(tree, i))
1341 && validate_fpdef(CHILD(tree, i+1)));
1342 i += 2;
1343 if (res && (nch - i) >= 2 && TYPE(CHILD(tree, i)) == EQUAL) {
1344 res = (validate_equal(CHILD(tree, i))
1345 && validate_test(CHILD(tree, i+1)));
1346 i += 2;
1347 }
1348 }
1349 if (res && nch - i != 0) {
1350 res = 0;
1351 err_string("illegal formation for varargslist");
1352 }
1353 }
1354 }
1355 return res;
1356 }
1357
1358
1359 /* list_iter: list_for | list_if
1360 */
1361 static int
validate_list_iter(node * tree)1362 validate_list_iter(node *tree)
1363 {
1364 int res = (validate_ntype(tree, list_iter)
1365 && validate_numnodes(tree, 1, "list_iter"));
1366 if (res && TYPE(CHILD(tree, 0)) == list_for)
1367 res = validate_list_for(CHILD(tree, 0));
1368 else
1369 res = validate_list_if(CHILD(tree, 0));
1370
1371 return res;
1372 }
1373
1374 /* comp_iter: comp_for | comp_if
1375 */
1376 static int
validate_comp_iter(node * tree)1377 validate_comp_iter(node *tree)
1378 {
1379 int res = (validate_ntype(tree, comp_iter)
1380 && validate_numnodes(tree, 1, "comp_iter"));
1381 if (res && TYPE(CHILD(tree, 0)) == comp_for)
1382 res = validate_comp_for(CHILD(tree, 0));
1383 else
1384 res = validate_comp_if(CHILD(tree, 0));
1385
1386 return res;
1387 }
1388
1389 /* list_for: 'for' exprlist 'in' testlist [list_iter]
1390 */
1391 static int
validate_list_for(node * tree)1392 validate_list_for(node *tree)
1393 {
1394 int nch = NCH(tree);
1395 int res;
1396
1397 if (nch == 5)
1398 res = validate_list_iter(CHILD(tree, 4));
1399 else
1400 res = validate_numnodes(tree, 4, "list_for");
1401
1402 if (res)
1403 res = (validate_name(CHILD(tree, 0), "for")
1404 && validate_exprlist(CHILD(tree, 1))
1405 && validate_name(CHILD(tree, 2), "in")
1406 && validate_testlist_safe(CHILD(tree, 3)));
1407
1408 return res;
1409 }
1410
1411 /* comp_for: 'for' exprlist 'in' test [comp_iter]
1412 */
1413 static int
validate_comp_for(node * tree)1414 validate_comp_for(node *tree)
1415 {
1416 int nch = NCH(tree);
1417 int res;
1418
1419 if (nch == 5)
1420 res = validate_comp_iter(CHILD(tree, 4));
1421 else
1422 res = validate_numnodes(tree, 4, "comp_for");
1423
1424 if (res)
1425 res = (validate_name(CHILD(tree, 0), "for")
1426 && validate_exprlist(CHILD(tree, 1))
1427 && validate_name(CHILD(tree, 2), "in")
1428 && validate_or_test(CHILD(tree, 3)));
1429
1430 return res;
1431 }
1432
1433 /* list_if: 'if' old_test [list_iter]
1434 */
1435 static int
validate_list_if(node * tree)1436 validate_list_if(node *tree)
1437 {
1438 int nch = NCH(tree);
1439 int res;
1440
1441 if (nch == 3)
1442 res = validate_list_iter(CHILD(tree, 2));
1443 else
1444 res = validate_numnodes(tree, 2, "list_if");
1445
1446 if (res)
1447 res = (validate_name(CHILD(tree, 0), "if")
1448 && validate_old_test(CHILD(tree, 1)));
1449
1450 return res;
1451 }
1452
1453 /* comp_if: 'if' old_test [comp_iter]
1454 */
1455 static int
validate_comp_if(node * tree)1456 validate_comp_if(node *tree)
1457 {
1458 int nch = NCH(tree);
1459 int res;
1460
1461 if (nch == 3)
1462 res = validate_comp_iter(CHILD(tree, 2));
1463 else
1464 res = validate_numnodes(tree, 2, "comp_if");
1465
1466 if (res)
1467 res = (validate_name(CHILD(tree, 0), "if")
1468 && validate_old_test(CHILD(tree, 1)));
1469
1470 return res;
1471 }
1472
1473 /* validate_fpdef()
1474 *
1475 * fpdef:
1476 * NAME
1477 * | '(' fplist ')'
1478 */
1479 static int
validate_fpdef(node * tree)1480 validate_fpdef(node *tree)
1481 {
1482 int nch = NCH(tree);
1483 int res = validate_ntype(tree, fpdef);
1484
1485 if (res) {
1486 if (nch == 1)
1487 res = validate_ntype(CHILD(tree, 0), NAME);
1488 else if (nch == 3)
1489 res = (validate_lparen(CHILD(tree, 0))
1490 && validate_fplist(CHILD(tree, 1))
1491 && validate_rparen(CHILD(tree, 2)));
1492 else
1493 res = validate_numnodes(tree, 1, "fpdef");
1494 }
1495 return (res);
1496 }
1497
1498
1499 static int
validate_fplist(node * tree)1500 validate_fplist(node *tree)
1501 {
1502 return (validate_repeating_list(tree, fplist,
1503 validate_fpdef, "fplist"));
1504 }
1505
1506
1507 /* simple_stmt | compound_stmt
1508 *
1509 */
1510 static int
validate_stmt(node * tree)1511 validate_stmt(node *tree)
1512 {
1513 int res = (validate_ntype(tree, stmt)
1514 && validate_numnodes(tree, 1, "stmt"));
1515
1516 if (res) {
1517 tree = CHILD(tree, 0);
1518
1519 if (TYPE(tree) == simple_stmt)
1520 res = validate_simple_stmt(tree);
1521 else
1522 res = validate_compound_stmt(tree);
1523 }
1524 return (res);
1525 }
1526
1527
1528 /* small_stmt (';' small_stmt)* [';'] NEWLINE
1529 *
1530 */
1531 static int
validate_simple_stmt(node * tree)1532 validate_simple_stmt(node *tree)
1533 {
1534 int nch = NCH(tree);
1535 int res = (validate_ntype(tree, simple_stmt)
1536 && (nch >= 2)
1537 && validate_small_stmt(CHILD(tree, 0))
1538 && validate_newline(CHILD(tree, nch - 1)));
1539
1540 if (nch < 2)
1541 res = validate_numnodes(tree, 2, "simple_stmt");
1542 --nch; /* forget the NEWLINE */
1543 if (res && is_even(nch))
1544 res = validate_semi(CHILD(tree, --nch));
1545 if (res && (nch > 2)) {
1546 int i;
1547
1548 for (i = 1; res && (i < nch); i += 2)
1549 res = (validate_semi(CHILD(tree, i))
1550 && validate_small_stmt(CHILD(tree, i + 1)));
1551 }
1552 return (res);
1553 }
1554
1555
1556 static int
validate_small_stmt(node * tree)1557 validate_small_stmt(node *tree)
1558 {
1559 int nch = NCH(tree);
1560 int res = validate_numnodes(tree, 1, "small_stmt");
1561
1562 if (res) {
1563 int ntype = TYPE(CHILD(tree, 0));
1564
1565 if ( (ntype == expr_stmt)
1566 || (ntype == print_stmt)
1567 || (ntype == del_stmt)
1568 || (ntype == pass_stmt)
1569 || (ntype == flow_stmt)
1570 || (ntype == import_stmt)
1571 || (ntype == global_stmt)
1572 || (ntype == assert_stmt)
1573 || (ntype == exec_stmt))
1574 res = validate_node(CHILD(tree, 0));
1575 else {
1576 res = 0;
1577 err_string("illegal small_stmt child type");
1578 }
1579 }
1580 else if (nch == 1) {
1581 res = 0;
1582 PyErr_Format(parser_error,
1583 "Unrecognized child node of small_stmt: %d.",
1584 TYPE(CHILD(tree, 0)));
1585 }
1586 return (res);
1587 }
1588
1589
1590 /* compound_stmt:
1591 * if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
1592 */
1593 static int
validate_compound_stmt(node * tree)1594 validate_compound_stmt(node *tree)
1595 {
1596 int res = (validate_ntype(tree, compound_stmt)
1597 && validate_numnodes(tree, 1, "compound_stmt"));
1598 int ntype;
1599
1600 if (!res)
1601 return (0);
1602
1603 tree = CHILD(tree, 0);
1604 ntype = TYPE(tree);
1605 if ( (ntype == if_stmt)
1606 || (ntype == while_stmt)
1607 || (ntype == for_stmt)
1608 || (ntype == try_stmt)
1609 || (ntype == with_stmt)
1610 || (ntype == funcdef)
1611 || (ntype == classdef)
1612 || (ntype == decorated))
1613 res = validate_node(tree);
1614 else {
1615 res = 0;
1616 PyErr_Format(parser_error,
1617 "Illegal compound statement type: %d.", TYPE(tree));
1618 }
1619 return (res);
1620 }
1621
1622 static int
validate_yield_or_testlist(node * tree)1623 validate_yield_or_testlist(node *tree)
1624 {
1625 if (TYPE(tree) == yield_expr)
1626 return validate_yield_expr(tree);
1627 else
1628 return validate_testlist(tree);
1629 }
1630
1631 static int
validate_expr_stmt(node * tree)1632 validate_expr_stmt(node *tree)
1633 {
1634 int j;
1635 int nch = NCH(tree);
1636 int res = (validate_ntype(tree, expr_stmt)
1637 && is_odd(nch)
1638 && validate_testlist(CHILD(tree, 0)));
1639
1640 if (res && nch == 3
1641 && TYPE(CHILD(tree, 1)) == augassign) {
1642 res = validate_numnodes(CHILD(tree, 1), 1, "augassign")
1643 && validate_yield_or_testlist(CHILD(tree, 2));
1644
1645 if (res) {
1646 char *s = STR(CHILD(CHILD(tree, 1), 0));
1647
1648 res = (strcmp(s, "+=") == 0
1649 || strcmp(s, "-=") == 0
1650 || strcmp(s, "*=") == 0
1651 || strcmp(s, "/=") == 0
1652 || strcmp(s, "//=") == 0
1653 || strcmp(s, "%=") == 0
1654 || strcmp(s, "&=") == 0
1655 || strcmp(s, "|=") == 0
1656 || strcmp(s, "^=") == 0
1657 || strcmp(s, "<<=") == 0
1658 || strcmp(s, ">>=") == 0
1659 || strcmp(s, "**=") == 0);
1660 if (!res)
1661 err_string("illegal augmented assignment operator");
1662 }
1663 }
1664 else {
1665 for (j = 1; res && (j < nch); j += 2)
1666 res = validate_equal(CHILD(tree, j))
1667 && validate_yield_or_testlist(CHILD(tree, j + 1));
1668 }
1669 return (res);
1670 }
1671
1672
1673 /* print_stmt:
1674 *
1675 * 'print' ( [ test (',' test)* [','] ]
1676 * | '>>' test [ (',' test)+ [','] ] )
1677 */
1678 static int
validate_print_stmt(node * tree)1679 validate_print_stmt(node *tree)
1680 {
1681 int nch = NCH(tree);
1682 int res = (validate_ntype(tree, print_stmt)
1683 && (nch > 0)
1684 && validate_name(CHILD(tree, 0), "print"));
1685
1686 if (res && nch > 1) {
1687 int sym = TYPE(CHILD(tree, 1));
1688 int i = 1;
1689 int allow_trailing_comma = 1;
1690
1691 if (sym == test)
1692 res = validate_test(CHILD(tree, i++));
1693 else {
1694 if (nch < 3)
1695 res = validate_numnodes(tree, 3, "print_stmt");
1696 else {
1697 res = (validate_ntype(CHILD(tree, i), RIGHTSHIFT)
1698 && validate_test(CHILD(tree, i+1)));
1699 i += 2;
1700 allow_trailing_comma = 0;
1701 }
1702 }
1703 if (res) {
1704 /* ... (',' test)* [','] */
1705 while (res && i+2 <= nch) {
1706 res = (validate_comma(CHILD(tree, i))
1707 && validate_test(CHILD(tree, i+1)));
1708 allow_trailing_comma = 1;
1709 i += 2;
1710 }
1711 if (res && !allow_trailing_comma)
1712 res = validate_numnodes(tree, i, "print_stmt");
1713 else if (res && i < nch)
1714 res = validate_comma(CHILD(tree, i));
1715 }
1716 }
1717 return (res);
1718 }
1719
1720
1721 static int
validate_del_stmt(node * tree)1722 validate_del_stmt(node *tree)
1723 {
1724 return (validate_numnodes(tree, 2, "del_stmt")
1725 && validate_name(CHILD(tree, 0), "del")
1726 && validate_exprlist(CHILD(tree, 1)));
1727 }
1728
1729
1730 static int
validate_return_stmt(node * tree)1731 validate_return_stmt(node *tree)
1732 {
1733 int nch = NCH(tree);
1734 int res = (validate_ntype(tree, return_stmt)
1735 && ((nch == 1) || (nch == 2))
1736 && validate_name(CHILD(tree, 0), "return"));
1737
1738 if (res && (nch == 2))
1739 res = validate_testlist(CHILD(tree, 1));
1740
1741 return (res);
1742 }
1743
1744
1745 static int
validate_raise_stmt(node * tree)1746 validate_raise_stmt(node *tree)
1747 {
1748 int nch = NCH(tree);
1749 int res = (validate_ntype(tree, raise_stmt)
1750 && ((nch == 1) || (nch == 2) || (nch == 4) || (nch == 6)));
1751
1752 if (res) {
1753 res = validate_name(CHILD(tree, 0), "raise");
1754 if (res && (nch >= 2))
1755 res = validate_test(CHILD(tree, 1));
1756 if (res && nch > 2) {
1757 res = (validate_comma(CHILD(tree, 2))
1758 && validate_test(CHILD(tree, 3)));
1759 if (res && (nch > 4))
1760 res = (validate_comma(CHILD(tree, 4))
1761 && validate_test(CHILD(tree, 5)));
1762 }
1763 }
1764 else
1765 (void) validate_numnodes(tree, 2, "raise");
1766 if (res && (nch == 4))
1767 res = (validate_comma(CHILD(tree, 2))
1768 && validate_test(CHILD(tree, 3)));
1769
1770 return (res);
1771 }
1772
1773
1774 /* yield_expr: 'yield' [testlist]
1775 */
1776 static int
validate_yield_expr(node * tree)1777 validate_yield_expr(node *tree)
1778 {
1779 int nch = NCH(tree);
1780 int res = (validate_ntype(tree, yield_expr)
1781 && ((nch == 1) || (nch == 2))
1782 && validate_name(CHILD(tree, 0), "yield"));
1783
1784 if (res && (nch == 2))
1785 res = validate_testlist(CHILD(tree, 1));
1786
1787 return (res);
1788 }
1789
1790
1791 /* yield_stmt: yield_expr
1792 */
1793 static int
validate_yield_stmt(node * tree)1794 validate_yield_stmt(node *tree)
1795 {
1796 return (validate_ntype(tree, yield_stmt)
1797 && validate_numnodes(tree, 1, "yield_stmt")
1798 && validate_yield_expr(CHILD(tree, 0)));
1799 }
1800
1801
1802 static int
validate_import_as_name(node * tree)1803 validate_import_as_name(node *tree)
1804 {
1805 int nch = NCH(tree);
1806 int ok = validate_ntype(tree, import_as_name);
1807
1808 if (ok) {
1809 if (nch == 1)
1810 ok = validate_name(CHILD(tree, 0), NULL);
1811 else if (nch == 3)
1812 ok = (validate_name(CHILD(tree, 0), NULL)
1813 && validate_name(CHILD(tree, 1), "as")
1814 && validate_name(CHILD(tree, 2), NULL));
1815 else
1816 ok = validate_numnodes(tree, 3, "import_as_name");
1817 }
1818 return ok;
1819 }
1820
1821
1822 /* dotted_name: NAME ("." NAME)*
1823 */
1824 static int
validate_dotted_name(node * tree)1825 validate_dotted_name(node *tree)
1826 {
1827 int nch = NCH(tree);
1828 int res = (validate_ntype(tree, dotted_name)
1829 && is_odd(nch)
1830 && validate_name(CHILD(tree, 0), NULL));
1831 int i;
1832
1833 for (i = 1; res && (i < nch); i += 2) {
1834 res = (validate_dot(CHILD(tree, i))
1835 && validate_name(CHILD(tree, i+1), NULL));
1836 }
1837 return res;
1838 }
1839
1840
1841 /* dotted_as_name: dotted_name [NAME NAME]
1842 */
1843 static int
validate_dotted_as_name(node * tree)1844 validate_dotted_as_name(node *tree)
1845 {
1846 int nch = NCH(tree);
1847 int res = validate_ntype(tree, dotted_as_name);
1848
1849 if (res) {
1850 if (nch == 1)
1851 res = validate_dotted_name(CHILD(tree, 0));
1852 else if (nch == 3)
1853 res = (validate_dotted_name(CHILD(tree, 0))
1854 && validate_name(CHILD(tree, 1), "as")
1855 && validate_name(CHILD(tree, 2), NULL));
1856 else {
1857 res = 0;
1858 err_string("illegal number of children for dotted_as_name");
1859 }
1860 }
1861 return res;
1862 }
1863
1864
1865 /* dotted_as_name (',' dotted_as_name)* */
1866 static int
validate_dotted_as_names(node * tree)1867 validate_dotted_as_names(node *tree)
1868 {
1869 int nch = NCH(tree);
1870 int res = is_odd(nch) && validate_dotted_as_name(CHILD(tree, 0));
1871 int i;
1872
1873 for (i = 1; res && (i < nch); i += 2)
1874 res = (validate_comma(CHILD(tree, i))
1875 && validate_dotted_as_name(CHILD(tree, i + 1)));
1876 return (res);
1877 }
1878
1879
1880 /* import_as_name (',' import_as_name)* [','] */
1881 static int
validate_import_as_names(node * tree)1882 validate_import_as_names(node *tree)
1883 {
1884 int nch = NCH(tree);
1885 int res = validate_import_as_name(CHILD(tree, 0));
1886 int i;
1887
1888 for (i = 1; res && (i + 1 < nch); i += 2)
1889 res = (validate_comma(CHILD(tree, i))
1890 && validate_import_as_name(CHILD(tree, i + 1)));
1891 return (res);
1892 }
1893
1894
1895 /* 'import' dotted_as_names */
1896 static int
validate_import_name(node * tree)1897 validate_import_name(node *tree)
1898 {
1899 return (validate_ntype(tree, import_name)
1900 && validate_numnodes(tree, 2, "import_name")
1901 && validate_name(CHILD(tree, 0), "import")
1902 && validate_dotted_as_names(CHILD(tree, 1)));
1903 }
1904
1905 /* Helper function to count the number of leading dots in
1906 * 'from ...module import name'
1907 */
1908 static int
count_from_dots(node * tree)1909 count_from_dots(node *tree)
1910 {
1911 int i;
1912 for (i = 1; i < NCH(tree); i++)
1913 if (TYPE(CHILD(tree, i)) != DOT)
1914 break;
1915 return i-1;
1916 }
1917
1918 /* import_from: ('from' ('.'* dotted_name | '.'+)
1919 * 'import' ('*' | '(' import_as_names ')' | import_as_names))
1920 */
1921 static int
validate_import_from(node * tree)1922 validate_import_from(node *tree)
1923 {
1924 int nch = NCH(tree);
1925 int ndots = count_from_dots(tree);
1926 int havename = (TYPE(CHILD(tree, ndots + 1)) == dotted_name);
1927 int offset = ndots + havename;
1928 int res = validate_ntype(tree, import_from)
1929 && (offset >= 1)
1930 && (nch >= 3 + offset)
1931 && validate_name(CHILD(tree, 0), "from")
1932 && (!havename || validate_dotted_name(CHILD(tree, ndots + 1)))
1933 && validate_name(CHILD(tree, offset + 1), "import");
1934
1935 if (res && TYPE(CHILD(tree, offset + 2)) == LPAR)
1936 res = ((nch == offset + 5)
1937 && validate_lparen(CHILD(tree, offset + 2))
1938 && validate_import_as_names(CHILD(tree, offset + 3))
1939 && validate_rparen(CHILD(tree, offset + 4)));
1940 else if (res && TYPE(CHILD(tree, offset + 2)) != STAR)
1941 res = validate_import_as_names(CHILD(tree, offset + 2));
1942 return (res);
1943 }
1944
1945
1946 /* import_stmt: import_name | import_from */
1947 static int
validate_import_stmt(node * tree)1948 validate_import_stmt(node *tree)
1949 {
1950 int nch = NCH(tree);
1951 int res = validate_numnodes(tree, 1, "import_stmt");
1952
1953 if (res) {
1954 int ntype = TYPE(CHILD(tree, 0));
1955
1956 if (ntype == import_name || ntype == import_from)
1957 res = validate_node(CHILD(tree, 0));
1958 else {
1959 res = 0;
1960 err_string("illegal import_stmt child type");
1961 }
1962 }
1963 else if (nch == 1) {
1964 res = 0;
1965 PyErr_Format(parser_error,
1966 "Unrecognized child node of import_stmt: %d.",
1967 TYPE(CHILD(tree, 0)));
1968 }
1969 return (res);
1970 }
1971
1972
1973
1974
1975 static int
validate_global_stmt(node * tree)1976 validate_global_stmt(node *tree)
1977 {
1978 int j;
1979 int nch = NCH(tree);
1980 int res = (validate_ntype(tree, global_stmt)
1981 && is_even(nch) && (nch >= 2));
1982
1983 if (!res && !PyErr_Occurred())
1984 err_string("illegal global statement");
1985
1986 if (res)
1987 res = (validate_name(CHILD(tree, 0), "global")
1988 && validate_ntype(CHILD(tree, 1), NAME));
1989 for (j = 2; res && (j < nch); j += 2)
1990 res = (validate_comma(CHILD(tree, j))
1991 && validate_ntype(CHILD(tree, j + 1), NAME));
1992
1993 return (res);
1994 }
1995
1996
1997 /* exec_stmt:
1998 *
1999 * 'exec' expr ['in' test [',' test]]
2000 */
2001 static int
validate_exec_stmt(node * tree)2002 validate_exec_stmt(node *tree)
2003 {
2004 int nch = NCH(tree);
2005 int res = (validate_ntype(tree, exec_stmt)
2006 && ((nch == 2) || (nch == 4) || (nch == 6))
2007 && validate_name(CHILD(tree, 0), "exec")
2008 && validate_expr(CHILD(tree, 1)));
2009
2010 if (!res && !PyErr_Occurred())
2011 err_string("illegal exec statement");
2012 if (res && (nch > 2))
2013 res = (validate_name(CHILD(tree, 2), "in")
2014 && validate_test(CHILD(tree, 3)));
2015 if (res && (nch == 6))
2016 res = (validate_comma(CHILD(tree, 4))
2017 && validate_test(CHILD(tree, 5)));
2018
2019 return (res);
2020 }
2021
2022
2023 /* assert_stmt:
2024 *
2025 * 'assert' test [',' test]
2026 */
2027 static int
validate_assert_stmt(node * tree)2028 validate_assert_stmt(node *tree)
2029 {
2030 int nch = NCH(tree);
2031 int res = (validate_ntype(tree, assert_stmt)
2032 && ((nch == 2) || (nch == 4))
2033 && (validate_name(CHILD(tree, 0), "assert"))
2034 && validate_test(CHILD(tree, 1)));
2035
2036 if (!res && !PyErr_Occurred())
2037 err_string("illegal assert statement");
2038 if (res && (nch > 2))
2039 res = (validate_comma(CHILD(tree, 2))
2040 && validate_test(CHILD(tree, 3)));
2041
2042 return (res);
2043 }
2044
2045
2046 static int
validate_while(node * tree)2047 validate_while(node *tree)
2048 {
2049 int nch = NCH(tree);
2050 int res = (validate_ntype(tree, while_stmt)
2051 && ((nch == 4) || (nch == 7))
2052 && validate_name(CHILD(tree, 0), "while")
2053 && validate_test(CHILD(tree, 1))
2054 && validate_colon(CHILD(tree, 2))
2055 && validate_suite(CHILD(tree, 3)));
2056
2057 if (res && (nch == 7))
2058 res = (validate_name(CHILD(tree, 4), "else")
2059 && validate_colon(CHILD(tree, 5))
2060 && validate_suite(CHILD(tree, 6)));
2061
2062 return (res);
2063 }
2064
2065
2066 static int
validate_for(node * tree)2067 validate_for(node *tree)
2068 {
2069 int nch = NCH(tree);
2070 int res = (validate_ntype(tree, for_stmt)
2071 && ((nch == 6) || (nch == 9))
2072 && validate_name(CHILD(tree, 0), "for")
2073 && validate_exprlist(CHILD(tree, 1))
2074 && validate_name(CHILD(tree, 2), "in")
2075 && validate_testlist(CHILD(tree, 3))
2076 && validate_colon(CHILD(tree, 4))
2077 && validate_suite(CHILD(tree, 5)));
2078
2079 if (res && (nch == 9))
2080 res = (validate_name(CHILD(tree, 6), "else")
2081 && validate_colon(CHILD(tree, 7))
2082 && validate_suite(CHILD(tree, 8)));
2083
2084 return (res);
2085 }
2086
2087
2088 /* try_stmt:
2089 * 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
2090 ['finally' ':' suite]
2091 * | 'try' ':' suite 'finally' ':' suite
2092 *
2093 */
2094 static int
validate_try(node * tree)2095 validate_try(node *tree)
2096 {
2097 int nch = NCH(tree);
2098 int pos = 3;
2099 int res = (validate_ntype(tree, try_stmt)
2100 && (nch >= 6) && ((nch % 3) == 0));
2101
2102 if (res)
2103 res = (validate_name(CHILD(tree, 0), "try")
2104 && validate_colon(CHILD(tree, 1))
2105 && validate_suite(CHILD(tree, 2))
2106 && validate_colon(CHILD(tree, nch - 2))
2107 && validate_suite(CHILD(tree, nch - 1)));
2108 else if (!PyErr_Occurred()) {
2109 const char* name = "except";
2110 if (TYPE(CHILD(tree, nch - 3)) != except_clause)
2111 name = STR(CHILD(tree, nch - 3));
2112
2113 PyErr_Format(parser_error,
2114 "Illegal number of children for try/%s node.", name);
2115 }
2116 /* Handle try/finally statement */
2117 if (res && (TYPE(CHILD(tree, pos)) == NAME) &&
2118 (strcmp(STR(CHILD(tree, pos)), "finally") == 0)) {
2119 res = (validate_numnodes(tree, 6, "try/finally")
2120 && validate_colon(CHILD(tree, 4))
2121 && validate_suite(CHILD(tree, 5)));
2122 return (res);
2123 }
2124 /* try/except statement: skip past except_clause sections */
2125 while (res && pos < nch && (TYPE(CHILD(tree, pos)) == except_clause)) {
2126 res = (validate_except_clause(CHILD(tree, pos))
2127 && validate_colon(CHILD(tree, pos + 1))
2128 && validate_suite(CHILD(tree, pos + 2)));
2129 pos += 3;
2130 }
2131 /* skip else clause */
2132 if (res && pos < nch && (TYPE(CHILD(tree, pos)) == NAME) &&
2133 (strcmp(STR(CHILD(tree, pos)), "else") == 0)) {
2134 res = (validate_colon(CHILD(tree, pos + 1))
2135 && validate_suite(CHILD(tree, pos + 2)));
2136 pos += 3;
2137 }
2138 if (res && pos < nch) {
2139 /* last clause must be a finally */
2140 res = (validate_name(CHILD(tree, pos), "finally")
2141 && validate_numnodes(tree, pos + 3, "try/except/finally")
2142 && validate_colon(CHILD(tree, pos + 1))
2143 && validate_suite(CHILD(tree, pos + 2)));
2144 }
2145 return (res);
2146 }
2147
2148
2149 static int
validate_except_clause(node * tree)2150 validate_except_clause(node *tree)
2151 {
2152 int nch = NCH(tree);
2153 int res = (validate_ntype(tree, except_clause)
2154 && ((nch == 1) || (nch == 2) || (nch == 4))
2155 && validate_name(CHILD(tree, 0), "except"));
2156
2157 if (res && (nch > 1))
2158 res = validate_test(CHILD(tree, 1));
2159 if (res && (nch == 4)) {
2160 if (TYPE(CHILD(tree, 2)) == NAME)
2161 res = validate_name(CHILD(tree, 2), "as");
2162 else
2163 res = validate_comma(CHILD(tree, 2));
2164 res = res && validate_test(CHILD(tree, 3));
2165 }
2166 return (res);
2167 }
2168
2169
2170 static int
validate_test(node * tree)2171 validate_test(node *tree)
2172 {
2173 int nch = NCH(tree);
2174 int res = validate_ntype(tree, test) && is_odd(nch);
2175
2176 if (res && (TYPE(CHILD(tree, 0)) == lambdef))
2177 res = ((nch == 1)
2178 && validate_lambdef(CHILD(tree, 0)));
2179 else if (res) {
2180 res = validate_or_test(CHILD(tree, 0));
2181 res = (res && (nch == 1 || (nch == 5 &&
2182 validate_name(CHILD(tree, 1), "if") &&
2183 validate_or_test(CHILD(tree, 2)) &&
2184 validate_name(CHILD(tree, 3), "else") &&
2185 validate_test(CHILD(tree, 4)))));
2186 }
2187 return (res);
2188 }
2189
2190 static int
validate_old_test(node * tree)2191 validate_old_test(node *tree)
2192 {
2193 int nch = NCH(tree);
2194 int res = validate_ntype(tree, old_test) && (nch == 1);
2195
2196 if (res && (TYPE(CHILD(tree, 0)) == old_lambdef))
2197 res = (validate_old_lambdef(CHILD(tree, 0)));
2198 else if (res) {
2199 res = (validate_or_test(CHILD(tree, 0)));
2200 }
2201 return (res);
2202 }
2203
2204 static int
validate_or_test(node * tree)2205 validate_or_test(node *tree)
2206 {
2207 int nch = NCH(tree);
2208 int res = validate_ntype(tree, or_test) && is_odd(nch);
2209
2210 if (res) {
2211 int pos;
2212 res = validate_and_test(CHILD(tree, 0));
2213 for (pos = 1; res && (pos < nch); pos += 2)
2214 res = (validate_name(CHILD(tree, pos), "or")
2215 && validate_and_test(CHILD(tree, pos + 1)));
2216 }
2217 return (res);
2218 }
2219
2220
2221 static int
validate_and_test(node * tree)2222 validate_and_test(node *tree)
2223 {
2224 int pos;
2225 int nch = NCH(tree);
2226 int res = (validate_ntype(tree, and_test)
2227 && is_odd(nch)
2228 && validate_not_test(CHILD(tree, 0)));
2229
2230 for (pos = 1; res && (pos < nch); pos += 2)
2231 res = (validate_name(CHILD(tree, pos), "and")
2232 && validate_not_test(CHILD(tree, 0)));
2233
2234 return (res);
2235 }
2236
2237
2238 static int
validate_not_test(node * tree)2239 validate_not_test(node *tree)
2240 {
2241 int nch = NCH(tree);
2242 int res = validate_ntype(tree, not_test) && ((nch == 1) || (nch == 2));
2243
2244 if (res) {
2245 if (nch == 2)
2246 res = (validate_name(CHILD(tree, 0), "not")
2247 && validate_not_test(CHILD(tree, 1)));
2248 else if (nch == 1)
2249 res = validate_comparison(CHILD(tree, 0));
2250 }
2251 return (res);
2252 }
2253
2254
2255 static int
validate_comparison(node * tree)2256 validate_comparison(node *tree)
2257 {
2258 int pos;
2259 int nch = NCH(tree);
2260 int res = (validate_ntype(tree, comparison)
2261 && is_odd(nch)
2262 && validate_expr(CHILD(tree, 0)));
2263
2264 for (pos = 1; res && (pos < nch); pos += 2)
2265 res = (validate_comp_op(CHILD(tree, pos))
2266 && validate_expr(CHILD(tree, pos + 1)));
2267
2268 return (res);
2269 }
2270
2271
2272 static int
validate_comp_op(node * tree)2273 validate_comp_op(node *tree)
2274 {
2275 int res = 0;
2276 int nch = NCH(tree);
2277
2278 if (!validate_ntype(tree, comp_op))
2279 return (0);
2280 if (nch == 1) {
2281 /*
2282 * Only child will be a terminal with a well-defined symbolic name
2283 * or a NAME with a string of either 'is' or 'in'
2284 */
2285 tree = CHILD(tree, 0);
2286 switch (TYPE(tree)) {
2287 case LESS:
2288 case GREATER:
2289 case EQEQUAL:
2290 case EQUAL:
2291 case LESSEQUAL:
2292 case GREATEREQUAL:
2293 case NOTEQUAL:
2294 res = 1;
2295 break;
2296 case NAME:
2297 res = ((strcmp(STR(tree), "in") == 0)
2298 || (strcmp(STR(tree), "is") == 0));
2299 if (!res) {
2300 PyErr_Format(parser_error,
2301 "illegal operator '%s'", STR(tree));
2302 }
2303 break;
2304 default:
2305 err_string("illegal comparison operator type");
2306 break;
2307 }
2308 }
2309 else if ((res = validate_numnodes(tree, 2, "comp_op")) != 0) {
2310 res = (validate_ntype(CHILD(tree, 0), NAME)
2311 && validate_ntype(CHILD(tree, 1), NAME)
2312 && (((strcmp(STR(CHILD(tree, 0)), "is") == 0)
2313 && (strcmp(STR(CHILD(tree, 1)), "not") == 0))
2314 || ((strcmp(STR(CHILD(tree, 0)), "not") == 0)
2315 && (strcmp(STR(CHILD(tree, 1)), "in") == 0))));
2316 if (!res && !PyErr_Occurred())
2317 err_string("unknown comparison operator");
2318 }
2319 return (res);
2320 }
2321
2322
2323 static int
validate_expr(node * tree)2324 validate_expr(node *tree)
2325 {
2326 int j;
2327 int nch = NCH(tree);
2328 int res = (validate_ntype(tree, expr)
2329 && is_odd(nch)
2330 && validate_xor_expr(CHILD(tree, 0)));
2331
2332 for (j = 2; res && (j < nch); j += 2)
2333 res = (validate_xor_expr(CHILD(tree, j))
2334 && validate_vbar(CHILD(tree, j - 1)));
2335
2336 return (res);
2337 }
2338
2339
2340 static int
validate_xor_expr(node * tree)2341 validate_xor_expr(node *tree)
2342 {
2343 int j;
2344 int nch = NCH(tree);
2345 int res = (validate_ntype(tree, xor_expr)
2346 && is_odd(nch)
2347 && validate_and_expr(CHILD(tree, 0)));
2348
2349 for (j = 2; res && (j < nch); j += 2)
2350 res = (validate_circumflex(CHILD(tree, j - 1))
2351 && validate_and_expr(CHILD(tree, j)));
2352
2353 return (res);
2354 }
2355
2356
2357 static int
validate_and_expr(node * tree)2358 validate_and_expr(node *tree)
2359 {
2360 int pos;
2361 int nch = NCH(tree);
2362 int res = (validate_ntype(tree, and_expr)
2363 && is_odd(nch)
2364 && validate_shift_expr(CHILD(tree, 0)));
2365
2366 for (pos = 1; res && (pos < nch); pos += 2)
2367 res = (validate_ampersand(CHILD(tree, pos))
2368 && validate_shift_expr(CHILD(tree, pos + 1)));
2369
2370 return (res);
2371 }
2372
2373
2374 static int
validate_chain_two_ops(node * tree,int (* termvalid)(node *),int op1,int op2)2375 validate_chain_two_ops(node *tree, int (*termvalid)(node *), int op1, int op2)
2376 {
2377 int pos = 1;
2378 int nch = NCH(tree);
2379 int res = (is_odd(nch)
2380 && (*termvalid)(CHILD(tree, 0)));
2381
2382 for ( ; res && (pos < nch); pos += 2) {
2383 if (TYPE(CHILD(tree, pos)) != op1)
2384 res = validate_ntype(CHILD(tree, pos), op2);
2385 if (res)
2386 res = (*termvalid)(CHILD(tree, pos + 1));
2387 }
2388 return (res);
2389 }
2390
2391
2392 static int
validate_shift_expr(node * tree)2393 validate_shift_expr(node *tree)
2394 {
2395 return (validate_ntype(tree, shift_expr)
2396 && validate_chain_two_ops(tree, validate_arith_expr,
2397 LEFTSHIFT, RIGHTSHIFT));
2398 }
2399
2400
2401 static int
validate_arith_expr(node * tree)2402 validate_arith_expr(node *tree)
2403 {
2404 return (validate_ntype(tree, arith_expr)
2405 && validate_chain_two_ops(tree, validate_term, PLUS, MINUS));
2406 }
2407
2408
2409 static int
validate_term(node * tree)2410 validate_term(node *tree)
2411 {
2412 int pos = 1;
2413 int nch = NCH(tree);
2414 int res = (validate_ntype(tree, term)
2415 && is_odd(nch)
2416 && validate_factor(CHILD(tree, 0)));
2417
2418 for ( ; res && (pos < nch); pos += 2)
2419 res = (((TYPE(CHILD(tree, pos)) == STAR)
2420 || (TYPE(CHILD(tree, pos)) == SLASH)
2421 || (TYPE(CHILD(tree, pos)) == DOUBLESLASH)
2422 || (TYPE(CHILD(tree, pos)) == PERCENT))
2423 && validate_factor(CHILD(tree, pos + 1)));
2424
2425 return (res);
2426 }
2427
2428
2429 /* factor:
2430 *
2431 * factor: ('+'|'-'|'~') factor | power
2432 */
2433 static int
validate_factor(node * tree)2434 validate_factor(node *tree)
2435 {
2436 int nch = NCH(tree);
2437 int res = (validate_ntype(tree, factor)
2438 && (((nch == 2)
2439 && ((TYPE(CHILD(tree, 0)) == PLUS)
2440 || (TYPE(CHILD(tree, 0)) == MINUS)
2441 || (TYPE(CHILD(tree, 0)) == TILDE))
2442 && validate_factor(CHILD(tree, 1)))
2443 || ((nch == 1)
2444 && validate_power(CHILD(tree, 0)))));
2445 return (res);
2446 }
2447
2448
2449 /* power:
2450 *
2451 * power: atom trailer* ('**' factor)*
2452 */
2453 static int
validate_power(node * tree)2454 validate_power(node *tree)
2455 {
2456 int pos = 1;
2457 int nch = NCH(tree);
2458 int res = (validate_ntype(tree, power) && (nch >= 1)
2459 && validate_atom(CHILD(tree, 0)));
2460
2461 while (res && (pos < nch) && (TYPE(CHILD(tree, pos)) == trailer))
2462 res = validate_trailer(CHILD(tree, pos++));
2463 if (res && (pos < nch)) {
2464 if (!is_even(nch - pos)) {
2465 err_string("illegal number of nodes for 'power'");
2466 return (0);
2467 }
2468 for ( ; res && (pos < (nch - 1)); pos += 2)
2469 res = (validate_doublestar(CHILD(tree, pos))
2470 && validate_factor(CHILD(tree, pos + 1)));
2471 }
2472 return (res);
2473 }
2474
2475
2476 static int
validate_atom(node * tree)2477 validate_atom(node *tree)
2478 {
2479 int pos;
2480 int nch = NCH(tree);
2481 int res = validate_ntype(tree, atom);
2482
2483 if (res && nch < 1)
2484 res = validate_numnodes(tree, nch+1, "atom");
2485 if (res) {
2486 switch (TYPE(CHILD(tree, 0))) {
2487 case LPAR:
2488 res = ((nch <= 3)
2489 && (validate_rparen(CHILD(tree, nch - 1))));
2490
2491 if (res && (nch == 3)) {
2492 if (TYPE(CHILD(tree, 1))==yield_expr)
2493 res = validate_yield_expr(CHILD(tree, 1));
2494 else
2495 res = validate_testlist_comp(CHILD(tree, 1));
2496 }
2497 break;
2498 case LSQB:
2499 if (nch == 2)
2500 res = validate_ntype(CHILD(tree, 1), RSQB);
2501 else if (nch == 3)
2502 res = (validate_listmaker(CHILD(tree, 1))
2503 && validate_ntype(CHILD(tree, 2), RSQB));
2504 else {
2505 res = 0;
2506 err_string("illegal list display atom");
2507 }
2508 break;
2509 case LBRACE:
2510 res = ((nch <= 3)
2511 && validate_ntype(CHILD(tree, nch - 1), RBRACE));
2512
2513 if (res && (nch == 3))
2514 res = validate_dictorsetmaker(CHILD(tree, 1));
2515 break;
2516 case BACKQUOTE:
2517 res = ((nch == 3)
2518 && validate_testlist1(CHILD(tree, 1))
2519 && validate_ntype(CHILD(tree, 2), BACKQUOTE));
2520 break;
2521 case NAME:
2522 case NUMBER:
2523 res = (nch == 1);
2524 break;
2525 case STRING:
2526 for (pos = 1; res && (pos < nch); ++pos)
2527 res = validate_ntype(CHILD(tree, pos), STRING);
2528 break;
2529 default:
2530 res = 0;
2531 break;
2532 }
2533 }
2534 return (res);
2535 }
2536
2537
2538 /* listmaker:
2539 * test ( list_for | (',' test)* [','] )
2540 */
2541 static int
validate_listmaker(node * tree)2542 validate_listmaker(node *tree)
2543 {
2544 int nch = NCH(tree);
2545 int ok = nch;
2546
2547 if (nch == 0)
2548 err_string("missing child nodes of listmaker");
2549 else
2550 ok = validate_test(CHILD(tree, 0));
2551
2552 /*
2553 * list_for | (',' test)* [',']
2554 */
2555 if (nch == 2 && TYPE(CHILD(tree, 1)) == list_for)
2556 ok = validate_list_for(CHILD(tree, 1));
2557 else {
2558 /* (',' test)* [','] */
2559 int i = 1;
2560 while (ok && nch - i >= 2) {
2561 ok = (validate_comma(CHILD(tree, i))
2562 && validate_test(CHILD(tree, i+1)));
2563 i += 2;
2564 }
2565 if (ok && i == nch-1)
2566 ok = validate_comma(CHILD(tree, i));
2567 else if (i != nch) {
2568 ok = 0;
2569 err_string("illegal trailing nodes for listmaker");
2570 }
2571 }
2572 return ok;
2573 }
2574
2575 /* testlist_comp:
2576 * test ( comp_for | (',' test)* [','] )
2577 */
2578 static int
validate_testlist_comp(node * tree)2579 validate_testlist_comp(node *tree)
2580 {
2581 int nch = NCH(tree);
2582 int ok = nch;
2583
2584 if (nch == 0)
2585 err_string("missing child nodes of testlist_comp");
2586 else {
2587 ok = validate_test(CHILD(tree, 0));
2588 }
2589
2590 /*
2591 * comp_for | (',' test)* [',']
2592 */
2593 if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for)
2594 ok = validate_comp_for(CHILD(tree, 1));
2595 else {
2596 /* (',' test)* [','] */
2597 int i = 1;
2598 while (ok && nch - i >= 2) {
2599 ok = (validate_comma(CHILD(tree, i))
2600 && validate_test(CHILD(tree, i+1)));
2601 i += 2;
2602 }
2603 if (ok && i == nch-1)
2604 ok = validate_comma(CHILD(tree, i));
2605 else if (i != nch) {
2606 ok = 0;
2607 err_string("illegal trailing nodes for testlist_comp");
2608 }
2609 }
2610 return ok;
2611 }
2612
2613 /* decorator:
2614 * '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
2615 */
2616 static int
validate_decorator(node * tree)2617 validate_decorator(node *tree)
2618 {
2619 int ok;
2620 int nch = NCH(tree);
2621 ok = (validate_ntype(tree, decorator) &&
2622 (nch == 3 || nch == 5 || nch == 6) &&
2623 validate_at(CHILD(tree, 0)) &&
2624 validate_dotted_name(CHILD(tree, 1)) &&
2625 validate_newline(RCHILD(tree, -1)));
2626
2627 if (ok && nch != 3) {
2628 ok = (validate_lparen(CHILD(tree, 2)) &&
2629 validate_rparen(RCHILD(tree, -2)));
2630
2631 if (ok && nch == 6)
2632 ok = validate_arglist(CHILD(tree, 3));
2633 }
2634
2635 return ok;
2636 }
2637
2638 /* decorators:
2639 * decorator+
2640 */
2641 static int
validate_decorators(node * tree)2642 validate_decorators(node *tree)
2643 {
2644 int i, nch, ok;
2645 nch = NCH(tree);
2646 ok = validate_ntype(tree, decorators) && nch >= 1;
2647
2648 for (i = 0; ok && i < nch; ++i)
2649 ok = validate_decorator(CHILD(tree, i));
2650
2651 return ok;
2652 }
2653
2654 /* with_item:
2655 * test ['as' expr]
2656 */
2657 static int
validate_with_item(node * tree)2658 validate_with_item(node *tree)
2659 {
2660 int nch = NCH(tree);
2661 int ok = (validate_ntype(tree, with_item)
2662 && (nch == 1 || nch == 3)
2663 && validate_test(CHILD(tree, 0)));
2664 if (ok && nch == 3)
2665 ok = (validate_name(CHILD(tree, 1), "as")
2666 && validate_expr(CHILD(tree, 2)));
2667 return ok;
2668 }
2669
2670 /* with_stmt:
2671 * 0 1 ... -2 -1
2672 * 'with' with_item (',' with_item)* ':' suite
2673 */
2674 static int
validate_with_stmt(node * tree)2675 validate_with_stmt(node *tree)
2676 {
2677 int i;
2678 int nch = NCH(tree);
2679 int ok = (validate_ntype(tree, with_stmt)
2680 && (nch % 2 == 0)
2681 && validate_name(CHILD(tree, 0), "with")
2682 && validate_colon(RCHILD(tree, -2))
2683 && validate_suite(RCHILD(tree, -1)));
2684 for (i = 1; ok && i < nch - 2; i += 2)
2685 ok = validate_with_item(CHILD(tree, i));
2686 return ok;
2687 }
2688
2689 /* funcdef:
2690 *
2691 * -5 -4 -3 -2 -1
2692 * 'def' NAME parameters ':' suite
2693 */
2694 static int
validate_funcdef(node * tree)2695 validate_funcdef(node *tree)
2696 {
2697 int nch = NCH(tree);
2698 int ok = (validate_ntype(tree, funcdef)
2699 && (nch == 5)
2700 && validate_name(RCHILD(tree, -5), "def")
2701 && validate_ntype(RCHILD(tree, -4), NAME)
2702 && validate_colon(RCHILD(tree, -2))
2703 && validate_parameters(RCHILD(tree, -3))
2704 && validate_suite(RCHILD(tree, -1)));
2705 return ok;
2706 }
2707
2708
2709 /* decorated
2710 * decorators (classdef | funcdef)
2711 */
2712 static int
validate_decorated(node * tree)2713 validate_decorated(node *tree)
2714 {
2715 int nch = NCH(tree);
2716 int ok = (validate_ntype(tree, decorated)
2717 && (nch == 2)
2718 && validate_decorators(RCHILD(tree, -2)));
2719 if (TYPE(RCHILD(tree, -1)) == funcdef)
2720 ok = ok && validate_funcdef(RCHILD(tree, -1));
2721 else
2722 ok = ok && validate_class(RCHILD(tree, -1));
2723 return ok;
2724 }
2725
2726 static int
validate_lambdef(node * tree)2727 validate_lambdef(node *tree)
2728 {
2729 int nch = NCH(tree);
2730 int res = (validate_ntype(tree, lambdef)
2731 && ((nch == 3) || (nch == 4))
2732 && validate_name(CHILD(tree, 0), "lambda")
2733 && validate_colon(CHILD(tree, nch - 2))
2734 && validate_test(CHILD(tree, nch - 1)));
2735
2736 if (res && (nch == 4))
2737 res = validate_varargslist(CHILD(tree, 1));
2738 else if (!res && !PyErr_Occurred())
2739 (void) validate_numnodes(tree, 3, "lambdef");
2740
2741 return (res);
2742 }
2743
2744
2745 static int
validate_old_lambdef(node * tree)2746 validate_old_lambdef(node *tree)
2747 {
2748 int nch = NCH(tree);
2749 int res = (validate_ntype(tree, old_lambdef)
2750 && ((nch == 3) || (nch == 4))
2751 && validate_name(CHILD(tree, 0), "lambda")
2752 && validate_colon(CHILD(tree, nch - 2))
2753 && validate_test(CHILD(tree, nch - 1)));
2754
2755 if (res && (nch == 4))
2756 res = validate_varargslist(CHILD(tree, 1));
2757 else if (!res && !PyErr_Occurred())
2758 (void) validate_numnodes(tree, 3, "old_lambdef");
2759
2760 return (res);
2761 }
2762
2763
2764 /* arglist:
2765 *
2766 * (argument ',')* (argument [','] | '*' test [',' '**' test] | '**' test)
2767 */
2768 static int
validate_arglist(node * tree)2769 validate_arglist(node *tree)
2770 {
2771 int nch = NCH(tree);
2772 int i = 0;
2773 int ok = 1;
2774
2775 if (nch <= 0)
2776 /* raise the right error from having an invalid number of children */
2777 return validate_numnodes(tree, nch + 1, "arglist");
2778
2779 if (nch > 1) {
2780 for (i=0; i<nch; i++) {
2781 if (TYPE(CHILD(tree, i)) == argument) {
2782 node *ch = CHILD(tree, i);
2783 if (NCH(ch) == 2 && TYPE(CHILD(ch, 1)) == comp_for) {
2784 err_string("need '(', ')' for generator expression");
2785 return 0;
2786 }
2787 }
2788 }
2789 }
2790
2791 while (ok && nch-i >= 2) {
2792 /* skip leading (argument ',') */
2793 ok = (validate_argument(CHILD(tree, i))
2794 && validate_comma(CHILD(tree, i+1)));
2795 if (ok)
2796 i += 2;
2797 else
2798 PyErr_Clear();
2799 }
2800 ok = 1;
2801 if (nch-i > 0) {
2802 /*
2803 * argument | '*' test [',' '**' test] | '**' test
2804 */
2805 int sym = TYPE(CHILD(tree, i));
2806
2807 if (sym == argument) {
2808 ok = validate_argument(CHILD(tree, i));
2809 if (ok && i+1 != nch) {
2810 err_string("illegal arglist specification"
2811 " (extra stuff on end)");
2812 ok = 0;
2813 }
2814 }
2815 else if (sym == STAR) {
2816 ok = validate_star(CHILD(tree, i));
2817 if (ok && (nch-i == 2))
2818 ok = validate_test(CHILD(tree, i+1));
2819 else if (ok && (nch-i == 5))
2820 ok = (validate_test(CHILD(tree, i+1))
2821 && validate_comma(CHILD(tree, i+2))
2822 && validate_doublestar(CHILD(tree, i+3))
2823 && validate_test(CHILD(tree, i+4)));
2824 else {
2825 err_string("illegal use of '*' in arglist");
2826 ok = 0;
2827 }
2828 }
2829 else if (sym == DOUBLESTAR) {
2830 if (nch-i == 2)
2831 ok = (validate_doublestar(CHILD(tree, i))
2832 && validate_test(CHILD(tree, i+1)));
2833 else {
2834 err_string("illegal use of '**' in arglist");
2835 ok = 0;
2836 }
2837 }
2838 else {
2839 err_string("illegal arglist specification");
2840 ok = 0;
2841 }
2842 }
2843 return (ok);
2844 }
2845
2846
2847
2848 /* argument:
2849 *
2850 * [test '='] test [comp_for]
2851 */
2852 static int
validate_argument(node * tree)2853 validate_argument(node *tree)
2854 {
2855 int nch = NCH(tree);
2856 int res = (validate_ntype(tree, argument)
2857 && ((nch == 1) || (nch == 2) || (nch == 3))
2858 && validate_test(CHILD(tree, 0)));
2859
2860 if (res && (nch == 2))
2861 res = validate_comp_for(CHILD(tree, 1));
2862 else if (res && (nch == 3))
2863 res = (validate_equal(CHILD(tree, 1))
2864 && validate_test(CHILD(tree, 2)));
2865
2866 return (res);
2867 }
2868
2869
2870
2871 /* trailer:
2872 *
2873 * '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
2874 */
2875 static int
validate_trailer(node * tree)2876 validate_trailer(node *tree)
2877 {
2878 int nch = NCH(tree);
2879 int res = validate_ntype(tree, trailer) && ((nch == 2) || (nch == 3));
2880
2881 if (res) {
2882 switch (TYPE(CHILD(tree, 0))) {
2883 case LPAR:
2884 res = validate_rparen(CHILD(tree, nch - 1));
2885 if (res && (nch == 3))
2886 res = validate_arglist(CHILD(tree, 1));
2887 break;
2888 case LSQB:
2889 res = (validate_numnodes(tree, 3, "trailer")
2890 && validate_subscriptlist(CHILD(tree, 1))
2891 && validate_ntype(CHILD(tree, 2), RSQB));
2892 break;
2893 case DOT:
2894 res = (validate_numnodes(tree, 2, "trailer")
2895 && validate_ntype(CHILD(tree, 1), NAME));
2896 break;
2897 default:
2898 res = 0;
2899 break;
2900 }
2901 }
2902 else {
2903 (void) validate_numnodes(tree, 2, "trailer");
2904 }
2905 return (res);
2906 }
2907
2908
2909 /* subscriptlist:
2910 *
2911 * subscript (',' subscript)* [',']
2912 */
2913 static int
validate_subscriptlist(node * tree)2914 validate_subscriptlist(node *tree)
2915 {
2916 return (validate_repeating_list(tree, subscriptlist,
2917 validate_subscript, "subscriptlist"));
2918 }
2919
2920
2921 /* subscript:
2922 *
2923 * '.' '.' '.' | test | [test] ':' [test] [sliceop]
2924 */
2925 static int
validate_subscript(node * tree)2926 validate_subscript(node *tree)
2927 {
2928 int offset = 0;
2929 int nch = NCH(tree);
2930 int res = validate_ntype(tree, subscript) && (nch >= 1) && (nch <= 4);
2931
2932 if (!res) {
2933 if (!PyErr_Occurred())
2934 err_string("invalid number of arguments for subscript node");
2935 return (0);
2936 }
2937 if (TYPE(CHILD(tree, 0)) == DOT)
2938 /* take care of ('.' '.' '.') possibility */
2939 return (validate_numnodes(tree, 3, "subscript")
2940 && validate_dot(CHILD(tree, 0))
2941 && validate_dot(CHILD(tree, 1))
2942 && validate_dot(CHILD(tree, 2)));
2943 if (nch == 1) {
2944 if (TYPE(CHILD(tree, 0)) == test)
2945 res = validate_test(CHILD(tree, 0));
2946 else
2947 res = validate_colon(CHILD(tree, 0));
2948 return (res);
2949 }
2950 /* Must be [test] ':' [test] [sliceop],
2951 * but at least one of the optional components will
2952 * be present, but we don't know which yet.
2953 */
2954 if ((TYPE(CHILD(tree, 0)) != COLON) || (nch == 4)) {
2955 res = validate_test(CHILD(tree, 0));
2956 offset = 1;
2957 }
2958 if (res)
2959 res = validate_colon(CHILD(tree, offset));
2960 if (res) {
2961 int rem = nch - ++offset;
2962 if (rem) {
2963 if (TYPE(CHILD(tree, offset)) == test) {
2964 res = validate_test(CHILD(tree, offset));
2965 ++offset;
2966 --rem;
2967 }
2968 if (res && rem)
2969 res = validate_sliceop(CHILD(tree, offset));
2970 }
2971 }
2972 return (res);
2973 }
2974
2975
2976 static int
validate_sliceop(node * tree)2977 validate_sliceop(node *tree)
2978 {
2979 int nch = NCH(tree);
2980 int res = ((nch == 1) || validate_numnodes(tree, 2, "sliceop"))
2981 && validate_ntype(tree, sliceop);
2982 if (!res && !PyErr_Occurred()) {
2983 res = validate_numnodes(tree, 1, "sliceop");
2984 }
2985 if (res)
2986 res = validate_colon(CHILD(tree, 0));
2987 if (res && (nch == 2))
2988 res = validate_test(CHILD(tree, 1));
2989
2990 return (res);
2991 }
2992
2993
2994 static int
validate_exprlist(node * tree)2995 validate_exprlist(node *tree)
2996 {
2997 return (validate_repeating_list(tree, exprlist,
2998 validate_expr, "exprlist"));
2999 }
3000
3001
3002 /*
3003 * dictorsetmaker:
3004 *
3005 * (test ':' test (comp_for | (',' test ':' test)* [','])) |
3006 * (test (comp_for | (',' test)* [',']))
3007 */
3008 static int
validate_dictorsetmaker(node * tree)3009 validate_dictorsetmaker(node *tree)
3010 {
3011 int nch = NCH(tree);
3012 int ok = validate_ntype(tree, dictorsetmaker);
3013 int i = 0;
3014 int check_trailing_comma = 0;
3015
3016 assert(nch > 0);
3017
3018 if (ok && (nch == 1 || TYPE(CHILD(tree, 1)) == COMMA)) {
3019 /* We got a set:
3020 * test (',' test)* [',']
3021 */
3022 ok = validate_test(CHILD(tree, i++));
3023 while (ok && nch - i >= 2) {
3024 ok = (validate_comma(CHILD(tree, i))
3025 && validate_test(CHILD(tree, i+1)));
3026 i += 2;
3027 }
3028 check_trailing_comma = 1;
3029 }
3030 else if (ok && TYPE(CHILD(tree, 1)) == comp_for) {
3031 /* We got a set comprehension:
3032 * test comp_for
3033 */
3034 ok = (validate_test(CHILD(tree, 0))
3035 && validate_comp_for(CHILD(tree, 1)));
3036 }
3037 else if (ok && NCH(tree) > 3 && TYPE(CHILD(tree, 3)) == comp_for) {
3038 /* We got a dict comprehension:
3039 * test ':' test comp_for
3040 */
3041 ok = (validate_test(CHILD(tree, 0))
3042 && validate_colon(CHILD(tree, 1))
3043 && validate_test(CHILD(tree, 2))
3044 && validate_comp_for(CHILD(tree, 3)));
3045 }
3046 else if (ok) {
3047 /* We got a dict:
3048 * test ':' test (',' test ':' test)* [',']
3049 */
3050 if (nch >= 3) {
3051 ok = (validate_test(CHILD(tree, i))
3052 && validate_colon(CHILD(tree, i+1))
3053 && validate_test(CHILD(tree, i+2)));
3054 i += 3;
3055 }
3056 else {
3057 ok = 0;
3058 err_string("illegal number of nodes for dictorsetmaker");
3059 }
3060
3061 while (ok && nch - i >= 4) {
3062 ok = (validate_comma(CHILD(tree, i))
3063 && validate_test(CHILD(tree, i+1))
3064 && validate_colon(CHILD(tree, i+2))
3065 && validate_test(CHILD(tree, i+3)));
3066 i += 4;
3067 }
3068 check_trailing_comma = 1;
3069 }
3070 if (ok && check_trailing_comma) {
3071 if (i == nch-1)
3072 ok = validate_comma(CHILD(tree, i));
3073 else if (i != nch) {
3074 ok = 0;
3075 err_string("illegal trailing nodes for dictorsetmaker");
3076 }
3077 }
3078
3079 return ok;
3080 }
3081
3082
3083 static int
validate_eval_input(node * tree)3084 validate_eval_input(node *tree)
3085 {
3086 int pos;
3087 int nch = NCH(tree);
3088 int res = (validate_ntype(tree, eval_input)
3089 && (nch >= 2)
3090 && validate_testlist(CHILD(tree, 0))
3091 && validate_ntype(CHILD(tree, nch - 1), ENDMARKER));
3092
3093 for (pos = 1; res && (pos < (nch - 1)); ++pos)
3094 res = validate_ntype(CHILD(tree, pos), NEWLINE);
3095
3096 return (res);
3097 }
3098
3099
3100 static int
validate_node(node * tree)3101 validate_node(node *tree)
3102 {
3103 int nch = 0; /* num. children on current node */
3104 int res = 1; /* result value */
3105 node* next = 0; /* node to process after this one */
3106
3107 while (res && (tree != 0)) {
3108 nch = NCH(tree);
3109 next = 0;
3110 switch (TYPE(tree)) {
3111 /*
3112 * Definition nodes.
3113 */
3114 case funcdef:
3115 res = validate_funcdef(tree);
3116 break;
3117 case with_stmt:
3118 res = validate_with_stmt(tree);
3119 break;
3120 case classdef:
3121 res = validate_class(tree);
3122 break;
3123 case decorated:
3124 res = validate_decorated(tree);
3125 break;
3126 /*
3127 * "Trivial" parse tree nodes.
3128 * (Why did I call these trivial?)
3129 */
3130 case stmt:
3131 res = validate_stmt(tree);
3132 break;
3133 case small_stmt:
3134 /*
3135 * expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt
3136 * | import_stmt | global_stmt | exec_stmt | assert_stmt
3137 */
3138 res = validate_small_stmt(tree);
3139 break;
3140 case flow_stmt:
3141 res = (validate_numnodes(tree, 1, "flow_stmt")
3142 && ((TYPE(CHILD(tree, 0)) == break_stmt)
3143 || (TYPE(CHILD(tree, 0)) == continue_stmt)
3144 || (TYPE(CHILD(tree, 0)) == yield_stmt)
3145 || (TYPE(CHILD(tree, 0)) == return_stmt)
3146 || (TYPE(CHILD(tree, 0)) == raise_stmt)));
3147 if (res)
3148 next = CHILD(tree, 0);
3149 else if (nch == 1)
3150 err_string("illegal flow_stmt type");
3151 break;
3152 case yield_stmt:
3153 res = validate_yield_stmt(tree);
3154 break;
3155 /*
3156 * Compound statements.
3157 */
3158 case simple_stmt:
3159 res = validate_simple_stmt(tree);
3160 break;
3161 case compound_stmt:
3162 res = validate_compound_stmt(tree);
3163 break;
3164 /*
3165 * Fundamental statements.
3166 */
3167 case expr_stmt:
3168 res = validate_expr_stmt(tree);
3169 break;
3170 case print_stmt:
3171 res = validate_print_stmt(tree);
3172 break;
3173 case del_stmt:
3174 res = validate_del_stmt(tree);
3175 break;
3176 case pass_stmt:
3177 res = (validate_numnodes(tree, 1, "pass")
3178 && validate_name(CHILD(tree, 0), "pass"));
3179 break;
3180 case break_stmt:
3181 res = (validate_numnodes(tree, 1, "break")
3182 && validate_name(CHILD(tree, 0), "break"));
3183 break;
3184 case continue_stmt:
3185 res = (validate_numnodes(tree, 1, "continue")
3186 && validate_name(CHILD(tree, 0), "continue"));
3187 break;
3188 case return_stmt:
3189 res = validate_return_stmt(tree);
3190 break;
3191 case raise_stmt:
3192 res = validate_raise_stmt(tree);
3193 break;
3194 case import_stmt:
3195 res = validate_import_stmt(tree);
3196 break;
3197 case import_name:
3198 res = validate_import_name(tree);
3199 break;
3200 case import_from:
3201 res = validate_import_from(tree);
3202 break;
3203 case global_stmt:
3204 res = validate_global_stmt(tree);
3205 break;
3206 case exec_stmt:
3207 res = validate_exec_stmt(tree);
3208 break;
3209 case assert_stmt:
3210 res = validate_assert_stmt(tree);
3211 break;
3212 case if_stmt:
3213 res = validate_if(tree);
3214 break;
3215 case while_stmt:
3216 res = validate_while(tree);
3217 break;
3218 case for_stmt:
3219 res = validate_for(tree);
3220 break;
3221 case try_stmt:
3222 res = validate_try(tree);
3223 break;
3224 case suite:
3225 res = validate_suite(tree);
3226 break;
3227 /*
3228 * Expression nodes.
3229 */
3230 case testlist:
3231 res = validate_testlist(tree);
3232 break;
3233 case yield_expr:
3234 res = validate_yield_expr(tree);
3235 break;
3236 case testlist1:
3237 res = validate_testlist1(tree);
3238 break;
3239 case test:
3240 res = validate_test(tree);
3241 break;
3242 case and_test:
3243 res = validate_and_test(tree);
3244 break;
3245 case not_test:
3246 res = validate_not_test(tree);
3247 break;
3248 case comparison:
3249 res = validate_comparison(tree);
3250 break;
3251 case exprlist:
3252 res = validate_exprlist(tree);
3253 break;
3254 case comp_op:
3255 res = validate_comp_op(tree);
3256 break;
3257 case expr:
3258 res = validate_expr(tree);
3259 break;
3260 case xor_expr:
3261 res = validate_xor_expr(tree);
3262 break;
3263 case and_expr:
3264 res = validate_and_expr(tree);
3265 break;
3266 case shift_expr:
3267 res = validate_shift_expr(tree);
3268 break;
3269 case arith_expr:
3270 res = validate_arith_expr(tree);
3271 break;
3272 case term:
3273 res = validate_term(tree);
3274 break;
3275 case factor:
3276 res = validate_factor(tree);
3277 break;
3278 case power:
3279 res = validate_power(tree);
3280 break;
3281 case atom:
3282 res = validate_atom(tree);
3283 break;
3284
3285 default:
3286 /* Hopefully never reached! */
3287 err_string("unrecognized node type");
3288 res = 0;
3289 break;
3290 }
3291 tree = next;
3292 }
3293 return (res);
3294 }
3295
3296
3297 static int
validate_expr_tree(node * tree)3298 validate_expr_tree(node *tree)
3299 {
3300 int res = validate_eval_input(tree);
3301
3302 if (!res && !PyErr_Occurred())
3303 err_string("could not validate expression tuple");
3304
3305 return (res);
3306 }
3307
3308
3309 /* file_input:
3310 * (NEWLINE | stmt)* ENDMARKER
3311 */
3312 static int
validate_file_input(node * tree)3313 validate_file_input(node *tree)
3314 {
3315 int j;
3316 int nch = NCH(tree) - 1;
3317 int res = ((nch >= 0)
3318 && validate_ntype(CHILD(tree, nch), ENDMARKER));
3319
3320 for (j = 0; res && (j < nch); ++j) {
3321 if (TYPE(CHILD(tree, j)) == stmt)
3322 res = validate_stmt(CHILD(tree, j));
3323 else
3324 res = validate_newline(CHILD(tree, j));
3325 }
3326 /* This stays in to prevent any internal failures from getting to the
3327 * user. Hopefully, this won't be needed. If a user reports getting
3328 * this, we have some debugging to do.
3329 */
3330 if (!res && !PyErr_Occurred())
3331 err_string("VALIDATION FAILURE: report this to the maintainer!");
3332
3333 return (res);
3334 }
3335
3336 static int
validate_encoding_decl(node * tree)3337 validate_encoding_decl(node *tree)
3338 {
3339 int nch = NCH(tree);
3340 int res = ((nch == 1)
3341 && validate_file_input(CHILD(tree, 0)));
3342
3343 if (!res && !PyErr_Occurred())
3344 err_string("Error Parsing encoding_decl");
3345
3346 return res;
3347 }
3348
3349 static PyObject*
3350 pickle_constructor = NULL;
3351
3352
3353 static PyObject*
parser__pickler(PyObject * self,PyObject * args)3354 parser__pickler(PyObject *self, PyObject *args)
3355 {
3356 NOTE(ARGUNUSED(self))
3357 PyObject *result = NULL;
3358 PyObject *st = NULL;
3359 PyObject *empty_dict = NULL;
3360
3361 if (PyArg_ParseTuple(args, "O!:_pickler", &PyST_Type, &st)) {
3362 PyObject *newargs;
3363 PyObject *tuple;
3364
3365 if ((empty_dict = PyDict_New()) == NULL)
3366 goto finally;
3367 if ((newargs = Py_BuildValue("Oi", st, 1)) == NULL)
3368 goto finally;
3369 tuple = parser_st2tuple((PyST_Object*)NULL, newargs, empty_dict);
3370 if (tuple != NULL) {
3371 result = Py_BuildValue("O(O)", pickle_constructor, tuple);
3372 Py_DECREF(tuple);
3373 }
3374 Py_DECREF(empty_dict);
3375 Py_DECREF(newargs);
3376 }
3377 finally:
3378 Py_XDECREF(empty_dict);
3379
3380 return (result);
3381 }
3382
3383
3384 /* Functions exported by this module. Most of this should probably
3385 * be converted into an ST object with methods, but that is better
3386 * done directly in Python, allowing subclasses to be created directly.
3387 * We'd really have to write a wrapper around it all anyway to allow
3388 * inheritance.
3389 */
3390 static PyMethodDef parser_functions[] = {
3391 {"ast2tuple", (PyCFunction)parser_ast2tuple, PUBLIC_METHOD_TYPE,
3392 PyDoc_STR("Creates a tuple-tree representation of an ST.")},
3393 {"ast2list", (PyCFunction)parser_ast2list, PUBLIC_METHOD_TYPE,
3394 PyDoc_STR("Creates a list-tree representation of an ST.")},
3395 {"compileast", (PyCFunction)parser_compileast,PUBLIC_METHOD_TYPE,
3396 PyDoc_STR("Compiles an ST object into a code object.")},
3397 {"compilest", (PyCFunction)parser_compilest, PUBLIC_METHOD_TYPE,
3398 PyDoc_STR("Compiles an ST object into a code object.")},
3399 {"expr", (PyCFunction)parser_expr, PUBLIC_METHOD_TYPE,
3400 PyDoc_STR("Creates an ST object from an expression.")},
3401 {"isexpr", (PyCFunction)parser_isexpr, PUBLIC_METHOD_TYPE,
3402 PyDoc_STR("Determines if an ST object was created from an expression.")},
3403 {"issuite", (PyCFunction)parser_issuite, PUBLIC_METHOD_TYPE,
3404 PyDoc_STR("Determines if an ST object was created from a suite.")},
3405 {"suite", (PyCFunction)parser_suite, PUBLIC_METHOD_TYPE,
3406 PyDoc_STR("Creates an ST object from a suite.")},
3407 {"sequence2ast", (PyCFunction)parser_tuple2ast, PUBLIC_METHOD_TYPE,
3408 PyDoc_STR("Creates an ST object from a tree representation.")},
3409 {"sequence2st", (PyCFunction)parser_tuple2st, PUBLIC_METHOD_TYPE,
3410 PyDoc_STR("Creates an ST object from a tree representation.")},
3411 {"st2tuple", (PyCFunction)parser_st2tuple, PUBLIC_METHOD_TYPE,
3412 PyDoc_STR("Creates a tuple-tree representation of an ST.")},
3413 {"st2list", (PyCFunction)parser_st2list, PUBLIC_METHOD_TYPE,
3414 PyDoc_STR("Creates a list-tree representation of an ST.")},
3415 {"tuple2ast", (PyCFunction)parser_tuple2ast, PUBLIC_METHOD_TYPE,
3416 PyDoc_STR("Creates an ST object from a tree representation.")},
3417 {"tuple2st", (PyCFunction)parser_tuple2st, PUBLIC_METHOD_TYPE,
3418 PyDoc_STR("Creates an ST object from a tree representation.")},
3419
3420 /* private stuff: support pickle module */
3421 {"_pickler", (PyCFunction)parser__pickler, METH_VARARGS,
3422 PyDoc_STR("Returns the pickle magic to allow ST objects to be pickled.")},
3423
3424 {NULL, NULL, 0, NULL}
3425 };
3426
3427
3428 PyMODINIT_FUNC initparser(void); /* supply a prototype */
3429
3430 PyMODINIT_FUNC
initparser(void)3431 initparser(void)
3432 {
3433 PyObject *module, *copyreg;
3434
3435 Py_TYPE(&PyST_Type) = &PyType_Type;
3436 module = Py_InitModule("parser", parser_functions);
3437 if (module == NULL)
3438 return;
3439
3440 if (parser_error == 0)
3441 parser_error = PyErr_NewException("parser.ParserError", NULL, NULL);
3442
3443 if (parser_error == 0)
3444 /* caller will check PyErr_Occurred() */
3445 return;
3446 /* CAUTION: The code next used to skip bumping the refcount on
3447 * parser_error. That's a disaster if initparser() gets called more
3448 * than once. By incref'ing, we ensure that each module dict that
3449 * gets created owns its reference to the shared parser_error object,
3450 * and the file static parser_error vrbl owns a reference too.
3451 */
3452 Py_INCREF(parser_error);
3453 if (PyModule_AddObject(module, "ParserError", parser_error) != 0)
3454 return;
3455
3456 Py_INCREF(&PyST_Type);
3457 PyModule_AddObject(module, "ASTType", (PyObject*)&PyST_Type);
3458 Py_INCREF(&PyST_Type);
3459 PyModule_AddObject(module, "STType", (PyObject*)&PyST_Type);
3460
3461 PyModule_AddStringConstant(module, "__copyright__",
3462 parser_copyright_string);
3463 PyModule_AddStringConstant(module, "__doc__",
3464 parser_doc_string);
3465 PyModule_AddStringConstant(module, "__version__",
3466 parser_version_string);
3467
3468 /* Register to support pickling.
3469 * If this fails, the import of this module will fail because an
3470 * exception will be raised here; should we clear the exception?
3471 */
3472 copyreg = PyImport_ImportModuleNoBlock("copy_reg");
3473 if (copyreg != NULL) {
3474 PyObject *func, *pickler;
3475
3476 func = PyObject_GetAttrString(copyreg, "pickle");
3477 pickle_constructor = PyObject_GetAttrString(module, "sequence2st");
3478 pickler = PyObject_GetAttrString(module, "_pickler");
3479 Py_XINCREF(pickle_constructor);
3480 if ((func != NULL) && (pickle_constructor != NULL)
3481 && (pickler != NULL)) {
3482 PyObject *res;
3483
3484 res = PyObject_CallFunctionObjArgs(func, &PyST_Type, pickler,
3485 pickle_constructor, NULL);
3486 Py_XDECREF(res);
3487 }
3488 Py_XDECREF(func);
3489 Py_XDECREF(pickle_constructor);
3490 Py_XDECREF(pickler);
3491 Py_DECREF(copyreg);
3492 }
3493 }
3494