Lines Matching defs:tok_state

31 struct tok_state {  struct
34     char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL */
35     char *cur;          /* Next character in buffer */
36     char *inp;          /* End of data in buffer */
37     int fp_interactive; /* If the file descriptor is interactive */
38     char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
39     char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
40     const char *end;    /* End of input buffer if buf != NULL */
41     const char *start;  /* Start of current token if not NULL */
42     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
44     FILE *fp;           /* Rest of input; NULL if tokenizing a string */
45     int tabsize;        /* Tab spacing */
46     int indent;         /* Current indentation index */
47     int indstack[MAXINDENT];            /* Stack of indents */
48     int atbol;          /* Nonzero if at begin of new line */
49     int pendin;         /* Pending indents (if > 0) or dedents (if < 0) */
50     const char *prompt, *nextprompt;          /* For interactive prompting */
51     int lineno;         /* Current line number */
52     int first_lineno;   /* First line of a single line or multi line string
54     int level;          /* () [] {} Parentheses nesting level */
56     char parenstack[MAXLEVEL];
57     int parenlinenostack[MAXLEVEL];
58     int parencolstack[MAXLEVEL];
59     PyObject *filename;
61     int altindstack[MAXINDENT];         /* Stack of alternate indents */
63     enum decoding_state decoding_state;
64     int decoding_erred;         /* whether erred in decoding  */
65     char *encoding;         /* Source encoding. */
89 extern struct tok_state *PyTokenizer_FromString(const char *, int);  argument