Searched refs:tokens (Results 1 – 2 of 2) sorted by relevance
348 struct token *tokens; in tokenise() local355 token_list = tokens = calloc((end - buffer) / 2, sizeof(struct token)); in tokenise()356 if (!tokens) { in tokenise()407 tokens[tix].line = lineno; in tokenise()420 tokens[tix].size = q - p; in tokenise()423 tokens[tix].content = malloc(tokens[tix].size + 1); in tokenise()424 if (!tokens[tix].content) { in tokenise()428 memcpy(tokens[tix].content, start, tokens[tix].size); in tokenise()429 tokens[tix].content[tokens[tix].size] = 0; in tokenise()434 if (islower(tokens[tix].content[0])) { in tokenise()[all …]
78 tokens = [ 'LPAR', 'RPAR', 'ID', 'EXC' ] + reserved variable in id_parser