1#!/usr/bin/env python 2 3# Copyright 2016 the V8 project authors. All rights reserved. 4# Copyright 2014 The Chromium Authors. All rights reserved. 5# Use of this source code is governed by a BSD-style license that can be 6# found in the LICENSE file. 7 8"""Given the output of -t commands from a ninja build for a gyp and GN generated 9build, report on differences between the command lines.""" 10 11 12import os 13import shlex 14import subprocess 15import sys 16 17 18# Must be in v8/. 19BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 20os.chdir(BASE_DIR) 21 22 23g_total_differences = 0 24 25 26def FindAndRemoveArgWithValue(command_line, argname): 27 """Given a command line as a list, remove and return the value of an option 28 that takes a value as a separate entry. 29 30 Modifies |command_line| in place. 31 """ 32 if argname not in command_line: 33 return '' 34 location = command_line.index(argname) 35 value = command_line[location + 1] 36 command_line[location:location + 2] = [] 37 return value 38 39 40def MergeSpacedArgs(command_line, argname): 41 """Combine all arguments |argname| with their values, separated by a space.""" 42 i = 0 43 result = [] 44 while i < len(command_line): 45 arg = command_line[i] 46 if arg == argname: 47 result.append(arg + ' ' + command_line[i + 1]) 48 i += 1 49 else: 50 result.append(arg) 51 i += 1 52 return result 53 54 55def NormalizeSymbolArguments(command_line): 56 """Normalize -g arguments. 57 58 If there's no -g args, it's equivalent to -g0. -g2 is equivalent to -g. 59 Modifies |command_line| in place. 60 """ 61 # Strip -g0 if there's no symbols. 62 have_some_symbols = False 63 for x in command_line: 64 if x.startswith('-g') and x != '-g0': 65 have_some_symbols = True 66 if not have_some_symbols and '-g0' in command_line: 67 command_line.remove('-g0') 68 69 # Rename -g2 to -g. 70 if '-g2' in command_line: 71 command_line[command_line.index('-g2')] = '-g' 72 73 74def GetFlags(lines, build_dir): 75 """Turn a list of command lines into a semi-structured dict.""" 76 is_win = sys.platform == 'win32' 77 flags_by_output = {} 78 for line in lines: 79 command_line = shlex.split(line.strip(), posix=not is_win)[1:] 80 81 output_name = FindAndRemoveArgWithValue(command_line, '-o') 82 dep_name = FindAndRemoveArgWithValue(command_line, '-MF') 83 84 NormalizeSymbolArguments(command_line) 85 86 command_line = MergeSpacedArgs(command_line, '-Xclang') 87 88 cc_file = [x for x in command_line if x.endswith('.cc') or 89 x.endswith('.c') or 90 x.endswith('.cpp')] 91 if len(cc_file) != 1: 92 print 'Skipping %s' % command_line 93 continue 94 assert len(cc_file) == 1 95 96 if is_win: 97 rsp_file = [x for x in command_line if x.endswith('.rsp')] 98 assert len(rsp_file) <= 1 99 if rsp_file: 100 rsp_file = os.path.join(build_dir, rsp_file[0][1:]) 101 with open(rsp_file, "r") as open_rsp_file: 102 command_line = shlex.split(open_rsp_file, posix=False) 103 104 defines = [x for x in command_line if x.startswith('-D')] 105 include_dirs = [x for x in command_line if x.startswith('-I')] 106 dash_f = [x for x in command_line if x.startswith('-f')] 107 warnings = \ 108 [x for x in command_line if x.startswith('/wd' if is_win else '-W')] 109 others = [x for x in command_line if x not in defines and \ 110 x not in include_dirs and \ 111 x not in dash_f and \ 112 x not in warnings and \ 113 x not in cc_file] 114 115 for index, value in enumerate(include_dirs): 116 if value == '-Igen': 117 continue 118 path = value[2:] 119 if not os.path.isabs(path): 120 path = os.path.join(build_dir, path) 121 include_dirs[index] = '-I' + os.path.normpath(path) 122 123 # GYP supports paths above the source root like <(DEPTH)/../foo while such 124 # paths are unsupported by gn. But gn allows to use system-absolute paths 125 # instead (paths that start with single '/'). Normalize all paths. 126 cc_file = [os.path.normpath(os.path.join(build_dir, cc_file[0]))] 127 128 # Filter for libFindBadConstructs.so having a relative path in one and 129 # absolute path in the other. 130 others_filtered = [] 131 for x in others: 132 if x.startswith('-Xclang ') and x.endswith('libFindBadConstructs.so'): 133 others_filtered.append( 134 '-Xclang ' + 135 os.path.join(os.getcwd(), 136 os.path.normpath( 137 os.path.join('out/gn_flags', x.split(' ', 1)[1])))) 138 elif x.startswith('-B'): 139 others_filtered.append( 140 '-B' + 141 os.path.join(os.getcwd(), 142 os.path.normpath(os.path.join('out/gn_flags', x[2:])))) 143 else: 144 others_filtered.append(x) 145 others = others_filtered 146 147 flags_by_output[cc_file[0]] = { 148 'output': output_name, 149 'depname': dep_name, 150 'defines': sorted(defines), 151 'include_dirs': sorted(include_dirs), # TODO(scottmg): This is wrong. 152 'dash_f': sorted(dash_f), 153 'warnings': sorted(warnings), 154 'other': sorted(others), 155 } 156 return flags_by_output 157 158 159def CompareLists(gyp, gn, name, dont_care_gyp=None, dont_care_gn=None): 160 """Return a report of any differences between gyp and gn lists, ignoring 161 anything in |dont_care_{gyp|gn}| respectively.""" 162 global g_total_differences 163 if not dont_care_gyp: 164 dont_care_gyp = [] 165 if not dont_care_gn: 166 dont_care_gn = [] 167 output = '' 168 if gyp[name] != gn[name]: 169 gyp_set = set(gyp[name]) 170 gn_set = set(gn[name]) 171 missing_in_gyp = gyp_set - gn_set 172 missing_in_gn = gn_set - gyp_set 173 missing_in_gyp -= set(dont_care_gyp) 174 missing_in_gn -= set(dont_care_gn) 175 if missing_in_gyp or missing_in_gn: 176 output += ' %s differ:\n' % name 177 if missing_in_gyp: 178 output += ' In gyp, but not in GN:\n %s' % '\n '.join( 179 sorted(missing_in_gyp)) + '\n' 180 g_total_differences += len(missing_in_gyp) 181 if missing_in_gn: 182 output += ' In GN, but not in gyp:\n %s' % '\n '.join( 183 sorted(missing_in_gn)) + '\n\n' 184 g_total_differences += len(missing_in_gn) 185 return output 186 187 188def Run(command_line): 189 """Run |command_line| as a subprocess and return stdout. Raises on error.""" 190 try: 191 return subprocess.check_output(command_line, shell=True) 192 except subprocess.CalledProcessError as e: 193 # Rescue the output we got until the exception happened. 194 print '#### Stdout: ####################################################' 195 print e.output 196 print '#################################################################' 197 raise 198 199 200def main(): 201 if len(sys.argv) < 4: 202 print ('usage: %s gn_outdir gyp_outdir gn_target ' 203 '[gyp_target1, gyp_target2, ...]' % __file__) 204 return 1 205 206 if len(sys.argv) == 4: 207 sys.argv.append(sys.argv[3]) 208 gn_out_dir = sys.argv[1] 209 print >> sys.stderr, 'Expecting gn outdir in %s...' % gn_out_dir 210 gn = Run('ninja -C %s -t commands %s' % (gn_out_dir, sys.argv[3])) 211 if sys.platform == 'win32': 212 # On Windows flags are stored in .rsp files which are created during build. 213 print >> sys.stderr, 'Building in %s...' % gn_out_dir 214 Run('ninja -C %s -d keeprsp %s' % (gn_out_dir, sys.argv[3])) 215 216 gyp_out_dir = sys.argv[2] 217 print >> sys.stderr, 'Expecting gyp outdir in %s...' % gyp_out_dir 218 gyp = Run('ninja -C %s -t commands %s' % (gyp_out_dir, " ".join(sys.argv[4:]))) 219 if sys.platform == 'win32': 220 # On Windows flags are stored in .rsp files which are created during build. 221 print >> sys.stderr, 'Building in %s...' % gyp_out_dir 222 Run('ninja -C %s -d keeprsp %s' % (gyp_out_dir, " ".join(sys.argv[4:]))) 223 224 all_gyp_flags = GetFlags(gyp.splitlines(), 225 os.path.join(os.getcwd(), gyp_out_dir)) 226 all_gn_flags = GetFlags(gn.splitlines(), 227 os.path.join(os.getcwd(), gn_out_dir)) 228 gyp_files = set(all_gyp_flags.keys()) 229 gn_files = set(all_gn_flags.keys()) 230 different_source_list = gyp_files != gn_files 231 if different_source_list: 232 print 'Different set of sources files:' 233 print ' In gyp, not in GN:\n %s' % '\n '.join( 234 sorted(gyp_files - gn_files)) 235 print ' In GN, not in gyp:\n %s' % '\n '.join( 236 sorted(gn_files - gyp_files)) 237 print '\nNote that flags will only be compared for files in both sets.\n' 238 file_list = gyp_files & gn_files 239 files_with_given_differences = {} 240 for filename in sorted(file_list): 241 gyp_flags = all_gyp_flags[filename] 242 gn_flags = all_gn_flags[filename] 243 differences = CompareLists(gyp_flags, gn_flags, 'dash_f') 244 differences += CompareLists(gyp_flags, gn_flags, 'defines') 245 differences += CompareLists(gyp_flags, gn_flags, 'include_dirs', 246 ['-I%s' % os.path.dirname(BASE_DIR)]) 247 differences += CompareLists(gyp_flags, gn_flags, 'warnings', 248 # More conservative warnings in GN we consider to be OK. 249 dont_care_gyp=[ 250 '/wd4091', # 'keyword' : ignored on left of 'type' when no variable 251 # is declared. 252 '/wd4456', # Declaration hides previous local declaration. 253 '/wd4457', # Declaration hides function parameter. 254 '/wd4458', # Declaration hides class member. 255 '/wd4459', # Declaration hides global declaration. 256 '/wd4702', # Unreachable code. 257 '/wd4800', # Forcing value to bool 'true' or 'false'. 258 '/wd4838', # Conversion from 'type' to 'type' requires a narrowing 259 # conversion. 260 ] if sys.platform == 'win32' else None, 261 dont_care_gn=[ 262 '-Wendif-labels', 263 '-Wextra', 264 '-Wsign-compare', 265 ] if not sys.platform == 'win32' else None) 266 differences += CompareLists(gyp_flags, gn_flags, 'other') 267 if differences: 268 files_with_given_differences.setdefault(differences, []).append(filename) 269 270 for diff, files in files_with_given_differences.iteritems(): 271 print '\n'.join(sorted(files)) 272 print diff 273 274 print 'Total differences:', g_total_differences 275 # TODO(scottmg): Return failure on difference once we're closer to identical. 276 return 0 277 278 279if __name__ == '__main__': 280 sys.exit(main()) 281