1import itertools 2 3 4def generate_tokens(tokens): 5 numbers = itertools.count(0) 6 for line in tokens: 7 line = line.strip() 8 9 if not line: 10 continue 11 if line.strip().startswith('#'): 12 continue 13 14 name = line.split()[0] 15 yield (name, next(numbers)) 16 17 yield ('N_TOKENS', next(numbers)) 18 yield ('NT_OFFSET', 256) 19 20 21def generate_opmap(tokens): 22 for line in tokens: 23 line = line.strip() 24 25 if not line: 26 continue 27 if line.strip().startswith('#'): 28 continue 29 30 pieces = line.split() 31 32 if len(pieces) != 2: 33 continue 34 35 name, op = pieces 36 yield (op.strip("'"), name) 37 38 # Yield independently <>. This is needed so it does not collide 39 # with the token generation in "generate_tokens" because if this 40 # symbol is included in Grammar/Tokens, it will collide with != 41 # as it has the same name (NOTEQUAL). 42 yield ('<>', 'NOTEQUAL') 43