• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 
2 /* Parser-tokenizer link implementation */
3 
4 #include "pgenheaders.h"
5 #include "tokenizer.h"
6 #include "node.h"
7 #include "grammar.h"
8 #include "parser.h"
9 #include "parsetok.h"
10 #include "errcode.h"
11 #include "graminit.h"
12 
13 int Py_TabcheckFlag;
14 
15 
16 /* Forward */
17 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
18 static void initerr(perrdetail *err_ret, const char* filename);
19 
20 /* Parse input coming from a string.  Return error code, print some errors. */
21 node *
PyParser_ParseString(const char * s,grammar * g,int start,perrdetail * err_ret)22 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
23 {
24     return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
25 }
26 
27 node *
PyParser_ParseStringFlags(const char * s,grammar * g,int start,perrdetail * err_ret,int flags)28 PyParser_ParseStringFlags(const char *s, grammar *g, int start,
29                           perrdetail *err_ret, int flags)
30 {
31     return PyParser_ParseStringFlagsFilename(s, NULL,
32                                              g, start, err_ret, flags);
33 }
34 
35 node *
PyParser_ParseStringFlagsFilename(const char * s,const char * filename,grammar * g,int start,perrdetail * err_ret,int flags)36 PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
37                           grammar *g, int start,
38                           perrdetail *err_ret, int flags)
39 {
40     int iflags = flags;
41     return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
42                                                err_ret, &iflags);
43 }
44 
45 node *
PyParser_ParseStringFlagsFilenameEx(const char * s,const char * filename,grammar * g,int start,perrdetail * err_ret,int * flags)46 PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
47                           grammar *g, int start,
48                           perrdetail *err_ret, int *flags)
49 {
50     struct tok_state *tok;
51 
52     initerr(err_ret, filename);
53 
54     if ((tok = PyTokenizer_FromString(s, start == file_input)) == NULL) {
55         err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
56         return NULL;
57     }
58 
59     tok->filename = filename ? filename : "<string>";
60     if (Py_TabcheckFlag || Py_VerboseFlag) {
61         tok->altwarning = (tok->filename != NULL);
62         if (Py_TabcheckFlag >= 2)
63             tok->alterror++;
64     }
65 
66     return parsetok(tok, g, start, err_ret, flags);
67 }
68 
69 /* Parse input coming from a file.  Return error code, print some errors. */
70 
71 node *
PyParser_ParseFile(FILE * fp,const char * filename,grammar * g,int start,char * ps1,char * ps2,perrdetail * err_ret)72 PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
73                    char *ps1, char *ps2, perrdetail *err_ret)
74 {
75     return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
76                                    err_ret, 0);
77 }
78 
79 node *
PyParser_ParseFileFlags(FILE * fp,const char * filename,grammar * g,int start,char * ps1,char * ps2,perrdetail * err_ret,int flags)80 PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
81                         char *ps1, char *ps2, perrdetail *err_ret, int flags)
82 {
83     int iflags = flags;
84     return PyParser_ParseFileFlagsEx(fp, filename, g, start, ps1, ps2, err_ret, &iflags);
85 }
86 
87 node *
PyParser_ParseFileFlagsEx(FILE * fp,const char * filename,grammar * g,int start,char * ps1,char * ps2,perrdetail * err_ret,int * flags)88 PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, grammar *g, int start,
89                           char *ps1, char *ps2, perrdetail *err_ret, int *flags)
90 {
91     struct tok_state *tok;
92 
93     initerr(err_ret, filename);
94 
95     if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
96         err_ret->error = E_NOMEM;
97         return NULL;
98     }
99     tok->filename = filename;
100     if (Py_TabcheckFlag || Py_VerboseFlag) {
101         tok->altwarning = (filename != NULL);
102         if (Py_TabcheckFlag >= 2)
103             tok->alterror++;
104     }
105 
106     return parsetok(tok, g, start, err_ret, flags);
107 }
108 
109 #if 0
110 static char with_msg[] =
111 "%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
112 
113 static char as_msg[] =
114 "%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
115 
116 static void
117 warn(const char *msg, const char *filename, int lineno)
118 {
119     if (filename == NULL)
120         filename = "<string>";
121     PySys_WriteStderr(msg, filename, lineno);
122 }
123 #endif
124 
125 /* Parse input coming from the given tokenizer structure.
126    Return error code. */
127 
128 static node *
parsetok(struct tok_state * tok,grammar * g,int start,perrdetail * err_ret,int * flags)129 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
130          int *flags)
131 {
132     parser_state *ps;
133     node *n;
134     int started = 0;
135 
136     if ((ps = PyParser_New(g, start)) == NULL) {
137         fprintf(stderr, "no mem for new parser\n");
138         err_ret->error = E_NOMEM;
139         PyTokenizer_Free(tok);
140         return NULL;
141     }
142 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
143     if (*flags & PyPARSE_PRINT_IS_FUNCTION) {
144         ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
145     }
146     if (*flags & PyPARSE_UNICODE_LITERALS) {
147         ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
148     }
149 
150 #endif
151 
152     for (;;) {
153         char *a, *b;
154         int type;
155         size_t len;
156         char *str;
157         int col_offset;
158 
159         type = PyTokenizer_Get(tok, &a, &b);
160         if (type == ERRORTOKEN) {
161             err_ret->error = tok->done;
162             break;
163         }
164         if (type == ENDMARKER && started) {
165             type = NEWLINE; /* Add an extra newline */
166             started = 0;
167             /* Add the right number of dedent tokens,
168                except if a certain flag is given --
169                codeop.py uses this. */
170             if (tok->indent &&
171                 !(*flags & PyPARSE_DONT_IMPLY_DEDENT))
172             {
173                 tok->pendin = -tok->indent;
174                 tok->indent = 0;
175             }
176         }
177         else
178             started = 1;
179         len = (a != NULL && b != NULL) ? b - a : 0;
180         str = (char *) PyObject_MALLOC(len + 1);
181         if (str == NULL) {
182             fprintf(stderr, "no mem for next token\n");
183             err_ret->error = E_NOMEM;
184             break;
185         }
186         if (len > 0)
187             strncpy(str, a, len);
188         str[len] = '\0';
189 
190 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
191 #endif
192         if (a != NULL && a >= tok->line_start) {
193             col_offset = a - tok->line_start;
194         }
195         else {
196             col_offset = -1;
197         }
198 
199         if ((err_ret->error =
200              PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,
201                                &(err_ret->expected))) != E_OK) {
202             if (err_ret->error != E_DONE) {
203                 PyObject_FREE(str);
204                 err_ret->token = type;
205             }
206             break;
207         }
208     }
209 
210     if (err_ret->error == E_DONE) {
211         n = ps->p_tree;
212         ps->p_tree = NULL;
213     }
214     else
215         n = NULL;
216 
217 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
218     *flags = ps->p_flags;
219 #endif
220     PyParser_Delete(ps);
221 
222     if (n == NULL) {
223         if (tok->lineno <= 1 && tok->done == E_EOF)
224             err_ret->error = E_EOF;
225         err_ret->lineno = tok->lineno;
226         if (tok->buf != NULL) {
227             char *text = NULL;
228             size_t len;
229             assert(tok->cur - tok->buf < INT_MAX);
230             err_ret->offset = (int)(tok->cur - tok->buf);
231             len = tok->inp - tok->buf;
232 #ifdef Py_USING_UNICODE
233             text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset);
234 
235 #endif
236             if (text == NULL) {
237                 text = (char *) PyObject_MALLOC(len + 1);
238                 if (text != NULL) {
239                     if (len > 0)
240                         strncpy(text, tok->buf, len);
241                     text[len] = '\0';
242                 }
243             }
244             err_ret->text = text;
245         }
246     } else if (tok->encoding != NULL) {
247         /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
248          * allocated using PyMem_
249          */
250         node* r = PyNode_New(encoding_decl);
251         if (r)
252             r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
253         if (!r || !r->n_str) {
254             err_ret->error = E_NOMEM;
255             if (r)
256                 PyObject_FREE(r);
257             n = NULL;
258             goto done;
259         }
260         strcpy(r->n_str, tok->encoding);
261         PyMem_FREE(tok->encoding);
262         tok->encoding = NULL;
263         r->n_nchildren = 1;
264         r->n_child = n;
265         n = r;
266     }
267 
268 done:
269     PyTokenizer_Free(tok);
270 
271     return n;
272 }
273 
274 static void
initerr(perrdetail * err_ret,const char * filename)275 initerr(perrdetail *err_ret, const char *filename)
276 {
277     err_ret->error = E_OK;
278     err_ret->filename = filename;
279     err_ret->lineno = 0;
280     err_ret->offset = 0;
281     err_ret->text = NULL;
282     err_ret->token = -1;
283     err_ret->expected = -1;
284 }
285