1#!/usr/bin/env python 2 3from __future__ import print_function 4from collections import OrderedDict 5from shutil import copyfile 6import argparse 7import json 8import os 9import pprint 10import re 11import subprocess 12import sys 13import tempfile 14 15 16def normalize(dict_var): 17 for k, v in dict_var.items(): 18 if isinstance(v, OrderedDict): 19 normalize(v) 20 elif isinstance(v, list): 21 for e in v: 22 if isinstance(e, OrderedDict): 23 normalize(e) 24 elif type(v) is unicode: 25 st = v.encode('utf-8') 26 if v != "0x0" and re.match(r"0x[0-9A-Fa-f]+", v): 27 dict_var[k] = u'0x{{.*}}' 28 elif os.path.isfile(v): 29 dict_var[k] = u'{{.*}}' 30 else: 31 splits = (v.split(u' ')) 32 out_splits = [] 33 for split in splits: 34 inner_splits = split.rsplit(u':',2) 35 if os.path.isfile(inner_splits[0]): 36 out_splits.append( 37 u'{{.*}}:%s:%s' 38 %(inner_splits[1], 39 inner_splits[2])) 40 continue 41 out_splits.append(split) 42 43 dict_var[k] = ' '.join(out_splits) 44 45def filter_json(dict_var, filters, out): 46 for k, v in dict_var.items(): 47 if type(v) is unicode: 48 st = v.encode('utf-8') 49 if st in filters: 50 out.append(dict_var) 51 break 52 elif isinstance(v, OrderedDict): 53 filter_json(v, filters, out) 54 elif isinstance(v, list): 55 for e in v: 56 if isinstance(e, OrderedDict): 57 filter_json(e, filters, out) 58 59 60def default_clang_path(): 61 guessed_clang = os.path.join(os.path.dirname(__file__), "clang") 62 if os.path.isfile(guessed_clang): 63 return guessed_clang 64 return None 65 66 67def main(): 68 parser = argparse.ArgumentParser() 69 parser.add_argument("--clang", help="The clang binary (could be a relative or absolute path)", 70 action="store", default=default_clang_path()) 71 parser.add_argument("--source", help="the source file(s). Without --update, the command used to generate the JSON " 72 "will be of the format <clang> -cc1 -ast-dump=json <opts> <source>", 73 action="store", nargs=argparse.ONE_OR_MORE, required=True) 74 parser.add_argument("--filters", help="comma separated list of AST filters. Ex: --filters=TypedefDecl,BuiltinType", 75 action="store", default='') 76 update_or_generate_group = parser.add_mutually_exclusive_group() 77 update_or_generate_group.add_argument("--update", help="Update the file in-place", action="store_true") 78 update_or_generate_group.add_argument("--opts", help="other options", 79 action="store", default='', type=str) 80 parser.add_argument("--update-manual", help="When using --update, also update files that do not have the " 81 "autogenerated disclaimer", action="store_true") 82 args = parser.parse_args() 83 84 if not args.source: 85 sys.exit("Specify the source file to give to clang.") 86 87 clang_binary = os.path.abspath(args.clang) 88 if not os.path.isfile(clang_binary): 89 sys.exit("clang binary specified not present.") 90 91 for src in args.source: 92 process_file(src, clang_binary, cmdline_filters=args.filters, 93 cmdline_opts=args.opts, do_update=args.update, 94 force_update=args.update_manual) 95 96 97def process_file(source_file, clang_binary, cmdline_filters, cmdline_opts, 98 do_update, force_update): 99 note_firstline = "// NOTE: CHECK lines have been autogenerated by " \ 100 "gen_ast_dump_json_test.py" 101 filters_line_prefix = "// using --filters=" 102 note = note_firstline 103 104 cmd = [clang_binary, "-cc1"] 105 if do_update: 106 # When updating the first line of the test must be a RUN: line 107 with open(source_file, "r") as srcf: 108 first_line = srcf.readline() 109 found_autogenerated_line = False 110 filters_line = None 111 for i, line in enumerate(srcf.readlines()): 112 if found_autogenerated_line: 113 # print("Filters line: '", line.rstrip(), "'", sep="") 114 if line.startswith(filters_line_prefix): 115 filters_line = line[len(filters_line_prefix):].rstrip() 116 break 117 if line.startswith(note_firstline): 118 found_autogenerated_line = True 119 # print("Found autogenerated disclaimer at line", i + 1) 120 if not found_autogenerated_line and not force_update: 121 print("Not updating", source_file, "since it is not autogenerated.", 122 file=sys.stderr) 123 return 124 if not cmdline_filters and filters_line: 125 cmdline_filters = filters_line 126 print("Inferred filters as '" + cmdline_filters + "'") 127 128 if "RUN: %clang_cc1 " not in first_line: 129 sys.exit("When using --update the first line of the input file must contain RUN: %clang_cc1") 130 clang_start = first_line.find("%clang_cc1") + len("%clang_cc1") 131 file_check_idx = first_line.rfind("| FileCheck") 132 if file_check_idx: 133 dump_cmd = first_line[clang_start:file_check_idx] 134 else: 135 dump_cmd = first_line[clang_start:] 136 print("Inferred run arguments as '", dump_cmd, "'", sep="") 137 options = dump_cmd.split() 138 if "-ast-dump=json" not in options: 139 sys.exit("ERROR: RUN: line does not contain -ast-dump=json") 140 if "%s" not in options: 141 sys.exit("ERROR: RUN: line does not contain %s") 142 options.remove("%s") 143 else: 144 options = cmdline_opts.split() 145 options.append("-ast-dump=json") 146 cmd.extend(options) 147 using_ast_dump_filter = any('ast-dump-filter' in arg for arg in cmd) 148 cmd.append(source_file) 149 print("Will run", cmd) 150 filters = set() 151 if cmdline_filters: 152 note += "\n" + filters_line_prefix + cmdline_filters 153 filters = set(cmdline_filters.split(',')) 154 print("Will use the following filters:", filters) 155 156 try: 157 json_str = subprocess.check_output(cmd) 158 except Exception as ex: 159 print("The clang command failed with %s" % ex) 160 return -1 161 162 out_asts = [] 163 if using_ast_dump_filter: 164 splits = re.split('Dumping .*:\n', json_str) 165 if len(splits) > 1: 166 for split in splits[1:]: 167 j = json.loads(split.decode('utf-8'), object_pairs_hook=OrderedDict) 168 normalize(j) 169 out_asts.append(j) 170 else: 171 j = json.loads(json_str.decode('utf-8'), object_pairs_hook=OrderedDict) 172 normalize(j) 173 174 if len(filters) == 0: 175 out_asts.append(j) 176 else: 177 #assert using_ast_dump_filter is False,\ 178 # "Does not support using compiler's ast-dump-filter "\ 179 # "and the tool's filter option at the same time yet." 180 181 filter_json(j, filters, out_asts) 182 183 with tempfile.NamedTemporaryFile("wb", delete=False) as f: 184 with open(source_file, "r") as srcf: 185 for line in srcf.readlines(): 186 # copy up to the note: 187 if line.rstrip() == note_firstline: 188 break 189 f.write(line) 190 f.write(note + "\n") 191 for out_ast in out_asts: 192 append_str = json.dumps(out_ast, indent=1, ensure_ascii=False) 193 out_str = '\n\n' 194 index = 0 195 for append_line in append_str.splitlines()[2:]: 196 if index == 0: 197 out_str += '// CHECK: %s\n' %(append_line.rstrip()) 198 index += 1 199 else: 200 out_str += '// CHECK-NEXT: %s\n' %(append_line.rstrip()) 201 202 f.write(out_str) 203 f.flush() 204 f.close() 205 if do_update: 206 print("Updating json appended source file to %s." % source_file) 207 copyfile(f.name, source_file) 208 else: 209 partition = source_file.rpartition('.') 210 dest_path = '%s-json%s%s' % (partition[0], partition[1], partition[2]) 211 print("Writing json appended source file to %s." % dest_path) 212 copyfile(f.name, dest_path) 213 os.remove(f.name) 214 return 0 215 216 217if __name__ == '__main__': 218 main() 219