1 2import yaml 3import pprint 4 5# Tokens mnemonic: 6# directive: % 7# document_start: --- 8# document_end: ... 9# alias: * 10# anchor: & 11# tag: ! 12# scalar _ 13# block_sequence_start: [[ 14# block_mapping_start: {{ 15# block_end: ]} 16# flow_sequence_start: [ 17# flow_sequence_end: ] 18# flow_mapping_start: { 19# flow_mapping_end: } 20# entry: , 21# key: ? 22# value: : 23 24_replaces = { 25 yaml.DirectiveToken: '%', 26 yaml.DocumentStartToken: '---', 27 yaml.DocumentEndToken: '...', 28 yaml.AliasToken: '*', 29 yaml.AnchorToken: '&', 30 yaml.TagToken: '!', 31 yaml.ScalarToken: '_', 32 yaml.BlockSequenceStartToken: '[[', 33 yaml.BlockMappingStartToken: '{{', 34 yaml.BlockEndToken: ']}', 35 yaml.FlowSequenceStartToken: '[', 36 yaml.FlowSequenceEndToken: ']', 37 yaml.FlowMappingStartToken: '{', 38 yaml.FlowMappingEndToken: '}', 39 yaml.BlockEntryToken: ',', 40 yaml.FlowEntryToken: ',', 41 yaml.KeyToken: '?', 42 yaml.ValueToken: ':', 43} 44 45def test_tokens(data_filename, tokens_filename, verbose=False): 46 tokens1 = [] 47 with open(tokens_filename, 'r') as file: 48 tokens2 = file.read().split() 49 try: 50 with open(data_filename, 'rb') as file: 51 for token in yaml.scan(file): 52 if not isinstance(token, (yaml.StreamStartToken, yaml.StreamEndToken)): 53 tokens1.append(_replaces[token.__class__]) 54 finally: 55 if verbose: 56 print("TOKENS1:", ' '.join(tokens1)) 57 print("TOKENS2:", ' '.join(tokens2)) 58 assert len(tokens1) == len(tokens2), (tokens1, tokens2) 59 for token1, token2 in zip(tokens1, tokens2): 60 assert token1 == token2, (token1, token2) 61 62test_tokens.unittest = ['.data', '.tokens'] 63 64def test_scanner(data_filename, canonical_filename, verbose=False): 65 for filename in [data_filename, canonical_filename]: 66 tokens = [] 67 try: 68 with open(filename, 'rb') as file: 69 for token in yaml.scan(file): 70 tokens.append(token.__class__.__name__) 71 finally: 72 if verbose: 73 pprint.pprint(tokens) 74 75test_scanner.unittest = ['.data', '.canonical'] 76 77if __name__ == '__main__': 78 import test_appliance 79 test_appliance.run(globals()) 80 81