1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18# Parses the output of parse_ltp_{make,make_install} and generates a 19# corresponding Android.mk. 20# 21# This process is split into two steps so this second step can later be replaced 22# with an Android.bp generator. 23 24import argparse 25import fileinput 26import json 27import os.path 28import re 29 30import make_parser 31import make_install_parser 32 33MAKE_DRY_RUN_FILE_NAME = 'make_dry_run.dump' 34MAKE_INSTALL_DRY_RUN_FILE_NAME = 'make_install_dry_run.dump' 35DISABLED_TESTS_FILE_NAME = 'disabled_tests.txt' 36DISABLED_LIBS_FILE_NAME = 'disabled_libs.txt' 37DISABLED_CFLAGS_FILE_NAME = 'disabled_cflags.txt' 38 39 40class BuildGenerator(object): 41 '''A class to parse make output and convert the result to Android.ltp.mk modules. 42 43 Attributes: 44 _bp_result: directory of list of strings for blueprint file keyed by target name 45 _mk_result: directory of list of strings for makefile keyed by target name 46 _custom_cflags: dict of string (module name) to lists of strings (cflags 47 to add for said module) 48 _unused_custom_cflags: set of strings; tracks the modules with custom 49 cflags that we haven't yet seen 50 _packages: list of strings of packages for package list file 51 ''' 52 53 def __init__(self, custom_cflags): 54 self._bp_result = {} 55 self._mk_result = {} 56 self._custom_cflags = custom_cflags 57 self._unused_custom_cflags = set(custom_cflags) 58 self._packages = [] 59 60 def UniqueKeepOrder(self, sequence): 61 '''Get a copy of list where items are unique and order is preserved. 62 63 Args: 64 sequence: a sequence, can be a list, tuple, or other iterable 65 66 Returns: 67 a list where items copied from input sequence are unique 68 and order is preserved. 69 ''' 70 seen = set() 71 return [x for x in sequence if not (x in seen or seen.add(x))] 72 73 def ReadCommentedText(self, file_path): 74 '''Read pound commented text file into a list of lines. 75 76 Comments or empty lines will be excluded 77 78 Args: 79 file_path: string 80 ''' 81 ret = set() 82 with open(file_path, 'r') as f: 83 lines = [line.strip() for line in f.readlines()] 84 ret = set([s for s in lines if s and not s.startswith('#')]) 85 86 return ret 87 88 def ArTargetToLibraryName(self, ar_target): 89 '''Convert ar target to library name. 90 91 Args: 92 ar_target: string 93 ''' 94 return os.path.basename(ar_target)[len('lib'):-len('.a')] 95 96 def BuildExecutable(self, cc_target, local_src_files, local_cflags, 97 local_c_includes, local_libraries, ltp_libs, 98 ltp_libs_used, ltp_names_used): 99 '''Build a test module. 100 101 Args: 102 cc_target: string 103 local_src_files: list of string 104 local_cflags: list of string 105 local_c_includes: list of string 106 local_libraries: list of string 107 ltp_libs: list of string 108 ltp_libs_used: set of string 109 ltp_names_used: set of string, set of already used cc_target basenames 110 ''' 111 base_name = os.path.basename(cc_target) 112 if base_name in ltp_names_used: 113 print 'ERROR: base name {} of cc_target {} already used. Skipping...'.format( 114 base_name, cc_target) 115 return 116 ltp_names_used.add(base_name) 117 118 if cc_target in self._custom_cflags: 119 local_cflags.extend(self._custom_cflags[cc_target]) 120 self._unused_custom_cflags.remove(cc_target) 121 122 # ltp_defaults already adds the include directory 123 local_c_includes = [i for i in local_c_includes if i != 'include'] 124 target_name = 'ltp_%s' % base_name 125 target_bp = [] 126 127 self._packages.append(target_name) 128 129 target_bp.append('cc_test {') 130 target_bp.append(' name: "%s",' % target_name) 131 target_bp.append(' stem: "%s",' % base_name) 132 target_bp.append(' defaults: ["ltp_test_defaults"],') 133 134 if len(local_src_files) == 1: 135 target_bp.append(' srcs: ["%s"],' % list(local_src_files)[0]) 136 else: 137 target_bp.append(' srcs: [') 138 for src in local_src_files: 139 target_bp.append(' "%s",' % src) 140 target_bp.append(' ],') 141 142 if len(local_cflags) == 1: 143 target_bp.append(' cflags: ["%s"],' % list(local_cflags)[0]) 144 elif len(local_cflags) > 1: 145 target_bp.append(' cflags: [') 146 for cflag in local_cflags: 147 target_bp.append(' "%s",' % cflag) 148 target_bp.append(' ],') 149 150 if len(local_c_includes) == 1: 151 target_bp.append(' local_include_dirs: ["%s"],' % list(local_c_includes)[0]) 152 elif len(local_c_includes) > 1: 153 target_bp.append(' local_include_dirs: [') 154 for d in local_c_includes: 155 target_bp.append(' "%s",' % d) 156 target_bp.append(' ],') 157 158 bionic_builtin_libs = set(['m', 'rt', 'pthread', 'util']) 159 filtered_libs = set(local_libraries).difference(bionic_builtin_libs) 160 161 static_libraries = set(i for i in local_libraries if i in ltp_libs) 162 if len(static_libraries) == 1: 163 target_bp.append(' static_libs: ["libltp_%s"],' % list(static_libraries)[0]) 164 elif len(static_libraries) > 1: 165 target_bp.append(' static_libs: [') 166 for lib in static_libraries: 167 target_bp.append(' "libltp_%s",' % lib) 168 target_bp.append(' ],') 169 170 for lib in static_libraries: 171 ltp_libs_used.add(lib) 172 173 shared_libraries = set(i for i in filtered_libs if i not in ltp_libs) 174 if len(shared_libraries) == 1: 175 target_bp.append(' shared_libs: ["lib%s"],' % list(shared_libraries)[0]) 176 elif len(shared_libraries) > 1: 177 target_bp.append(' shared_libs: [') 178 for lib in shared_libraries: 179 target_bp.append(' "lib%s",' % lib) 180 target_bp.append(' ],') 181 182 target_bp.append('}') 183 target_bp.append('') 184 self._bp_result[target_name] = target_bp 185 186 def BuildStaticLibrary(self, ar_target, local_src_files, local_cflags, 187 local_c_includes): 188 '''Build a library module. 189 190 Args: 191 ar_target: string 192 local_src_files: list of string 193 local_cflags: list of string 194 local_c_includes: list of string 195 ''' 196 target_name = 'libltp_%s' % self.ArTargetToLibraryName(ar_target) 197 target_bp = [] 198 target_bp.append('cc_library_static {') 199 target_bp.append(' name: "%s",' % target_name) 200 target_bp.append(' defaults: ["ltp_defaults"],') 201 202 if len(local_c_includes): 203 target_bp.append(' local_include_dirs: [') 204 for d in local_c_includes: 205 target_bp.append(' "%s",' % d) 206 target_bp.append(' ],') 207 208 if len(local_cflags): 209 target_bp.append(' cflags: [') 210 for cflag in local_cflags: 211 target_bp.append(' "%s",' % cflag) 212 target_bp.append(' ],') 213 214 target_bp.append(' srcs: [') 215 for src in local_src_files: 216 target_bp.append(' "%s",' % src) 217 target_bp.append(' ],') 218 219 target_bp.append('}') 220 target_bp.append('') 221 self._bp_result[target_name] = target_bp 222 223 def BuildShellScript(self, install_target, local_src_file): 224 '''Build a shell script. 225 226 Args: 227 install_target: string 228 local_src_file: string 229 ''' 230 base_name = os.path.basename(install_target) 231 bp_result = [] 232 233 module = 'ltp_%s' % install_target.replace('/', '_') 234 self._packages.append(module) 235 236 module_dir = os.path.dirname(install_target) 237 module_stem = os.path.basename(install_target) 238 239 bp_result.append('sh_test {') 240 bp_result.append(' name: "%s",' % module) 241 bp_result.append(' src: "%s",' % local_src_file) 242 bp_result.append(' sub_dir: "ltp/%s",' % module_dir) 243 bp_result.append(' filename: "%s",' % module_stem) 244 bp_result.append(' compile_multilib: "both",') 245 bp_result.append('}') 246 bp_result.append('') 247 248 self._bp_result[module] = bp_result 249 250 def BuildPrebuilt(self, install_target, local_src_file): 251 '''Build a prebuild module. 252 253 Args: 254 install_target: string 255 local_src_file: string 256 ''' 257 base_name = os.path.basename(install_target) 258 mk_result = [] 259 mk_result.append('module_prebuilt := %s' % install_target) 260 mk_result.append('module_src_files := %s' % local_src_file) 261 module_dir = os.path.dirname(install_target) 262 module_stem = os.path.basename(install_target) 263 module = 'ltp_%s' % install_target.replace('/', '_') 264 self._packages.append(module) 265 266 mk_result.append('include $(ltp_build_prebuilt)') 267 mk_result.append('') 268 self._mk_result[base_name] = mk_result 269 270 def HandleParsedRule(self, line, rules): 271 '''Prepare parse rules. 272 273 Args: 274 line: string 275 rules: dictionary {string, dictionary} 276 ''' 277 groups = re.match(r'(.*)\[\'(.*)\'\] = \[(.*)\]', line).groups() 278 rule = groups[0] 279 rule_key = groups[1] 280 if groups[2] == '': 281 rule_value = [] 282 else: 283 rule_value = list(i.strip()[1:-1] for i in groups[2].split(',')) 284 285 rule_value = self.UniqueKeepOrder(rule_value) 286 rules.setdefault(rule, {})[rule_key] = rule_value 287 288 def ParseInput(self, input_list, ltp_root): 289 '''Parse a interpreted make output and produce Android.ltp.mk module. 290 291 Args: 292 input_list: list of string 293 ''' 294 disabled_tests = self.ReadCommentedText(DISABLED_TESTS_FILE_NAME) 295 disabled_libs = self.ReadCommentedText(DISABLED_LIBS_FILE_NAME) 296 disabled_cflags = self.ReadCommentedText(DISABLED_CFLAGS_FILE_NAME) 297 298 rules = {} 299 for line in input_list: 300 self.HandleParsedRule(line.strip(), rules) 301 302 # .a target -> .o files 303 ar = rules.get('ar', {}) 304 # executable target -> .o files 305 cc_link = rules.get('cc_link', {}) 306 # .o target -> .c file 307 cc_compile = rules.get('cc_compile', {}) 308 # executable target -> .c files 309 cc_compilelink = rules.get('cc_compilelink', {}) 310 # Target name -> CFLAGS passed to gcc 311 cc_flags = rules.get('cc_flags', {}) 312 # Target name -> -I paths passed to gcc 313 cc_includes = rules.get('cc_includes', {}) 314 # Target name -> -l paths passed to gcc 315 cc_libraries = rules.get('cc_libraries', {}) 316 # target -> prebuilt source 317 install = rules.get('install', {}) 318 319 # All libraries used by any LTP test (built or not) 320 ltp_libs = set(self.ArTargetToLibraryName(i) for i in ar.keys()) 321 # All libraries used by the LTP tests we actually build 322 ltp_libs_used = set() 323 ltp_names_used = set() 324 325 # Remove -Wno-error from cflags, we don't want to print warnings. 326 # Silence individual warnings in ltp_defaults or fix them. 327 for target in cc_flags: 328 if '-Wno-error' in cc_flags[target]: 329 cc_flags[target].remove('-Wno-error') 330 331 print( 332 "Disabled lib tests: Test cases listed here are" 333 "suggested to be disabled since they require a disabled library. " 334 "Please copy and paste them into disabled_tests.txt\n") 335 for i in cc_libraries: 336 if len(set(cc_libraries[i]).intersection(disabled_libs)) > 0: 337 if not os.path.basename(i) in disabled_tests: 338 print os.path.basename(i) 339 340 print("Disabled_cflag tests: Test cases listed here are" 341 "suggested to be disabled since they require a disabled cflag. " 342 "Please copy and paste them into disabled_tests.txt\n") 343 for i in cc_flags: 344 if len(set(cc_flags[i]).intersection(disabled_cflags)) > 0: 345 module_name = os.path.basename(i) 346 idx = module_name.find('_') 347 if idx > 0: 348 module_name = module_name[:idx] 349 print module_name 350 351 # Remove include directories that don't exist. They're an error in 352 # Soong. 353 for target in cc_includes: 354 cc_includes[target] = [i for i in cc_includes[target] if os.path.isdir(os.path.join(ltp_root, i))] 355 356 for target in cc_compilelink: 357 module_name = os.path.basename(target) 358 if module_name in disabled_tests: 359 continue 360 local_src_files = [] 361 src_files = cc_compilelink[target] 362 for i in src_files: 363 # some targets may have a mix of .c and .o files in srcs 364 # find the .c files to build those .o from cc_compile targets 365 if i.endswith('.o'): 366 local_src_files.extend(cc_compile[i]) 367 else: 368 local_src_files.append(i) 369 local_cflags = cc_flags[target] 370 local_c_includes = cc_includes[target] 371 local_libraries = cc_libraries[target] 372 if len(set(local_libraries).intersection(disabled_libs)) > 0: 373 continue 374 if len(set(local_cflags).intersection(disabled_cflags)) > 0: 375 continue 376 self.BuildExecutable(target, local_src_files, local_cflags, 377 local_c_includes, local_libraries, ltp_libs, 378 ltp_libs_used, ltp_names_used) 379 380 for target in cc_link: 381 if os.path.basename(target) in disabled_tests: 382 continue 383 local_src_files = set() 384 local_cflags = set() 385 local_c_includes = set() 386 local_libraries = cc_libraries[target] 387 # Accumulate flags for all .c files needed to build the .o files. 388 # (Android.mk requires a consistent set of flags across a given target. 389 # Thankfully using the superset of all flags in the target works fine 390 # with LTP tests.) 391 for obj in cc_link[target]: 392 for i in cc_compile[obj]: 393 local_src_files.add(i) 394 for i in cc_flags[obj]: 395 local_cflags.add(i) 396 for i in cc_includes[obj]: 397 local_c_includes.add(i) 398 if len(set(local_libraries).intersection(disabled_libs)) > 0: 399 continue 400 if len(set(local_cflags).intersection(disabled_cflags)) > 0: 401 continue 402 403 self.BuildExecutable(target, local_src_files, local_cflags, 404 local_c_includes, local_libraries, ltp_libs, 405 ltp_libs_used, ltp_names_used) 406 407 for target in ar: 408 # Disabled ltp library is already excluded 409 # since it won't be in ltp_libs_used 410 if not self.ArTargetToLibraryName(target) in ltp_libs_used: 411 continue 412 413 local_src_files = set() 414 local_cflags = set() 415 local_c_includes = set() 416 417 # TODO: disabled cflags 418 419 for obj in ar[target]: 420 for i in cc_compile[obj]: 421 local_src_files.add(i) 422 for i in cc_flags[obj]: 423 local_cflags.add(i) 424 for i in cc_includes[obj]: 425 local_c_includes.add(i) 426 427 if len(set(local_cflags).intersection(disabled_cflags)) > 0: 428 continue 429 430 local_src_files = sorted(local_src_files) 431 local_cflags = sorted(local_cflags) 432 local_c_includes = sorted(local_c_includes) 433 434 self.BuildStaticLibrary(target, local_src_files, local_cflags, 435 local_c_includes) 436 437 for target in install: 438 # Check if the absolute path to the prebuilt (relative to LTP_ROOT) 439 # is disabled. This is helpful in case there are duplicates with basename 440 # of the prebuilt. 441 # e.g. 442 # ./ testcases / kernel / fs / fs_bind / move / test01 443 # ./ testcases / kernel / fs / fs_bind / cloneNS / test01 444 # ./ testcases / kernel / fs / fs_bind / regression / test01 445 # ./ testcases / kernel / fs / fs_bind / rbind / test01 446 # ./ testcases / kernel / fs / fs_bind / bind / test01 447 if target in disabled_tests: 448 continue 449 if os.path.basename(target) in disabled_tests: 450 continue 451 local_src_files = install[target] 452 assert len(local_src_files) == 1 453 454 if target.startswith("testcases/bin/"): 455 self.BuildShellScript(target, local_src_files[0]) 456 else: 457 self.BuildPrebuilt(target, local_src_files[0]) 458 459 def WriteAndroidBp(self, output_path): 460 '''Write parse result to blueprint file. 461 462 Args: 463 output_path: string 464 ''' 465 with open(output_path, 'a') as f: 466 for k in sorted(self._bp_result.iterkeys()): 467 f.write('\n'.join(self._bp_result[k])) 468 f.write('\n') 469 self._bp_result = {} 470 471 def WriteAndroidMk(self, output_path): 472 '''Write parse result to make file. 473 474 Args: 475 output_path: string 476 ''' 477 with open(output_path, 'a') as f: 478 for k in sorted(self._mk_result.iterkeys()): 479 f.write('\n'.join(self._mk_result[k])) 480 f.write('\n') 481 self._mk_result = {} 482 483 def WritePackageList(self, output_path): 484 '''Write parse result to package list file. 485 486 Args: 487 output_path: string 488 ''' 489 with open(output_path, 'a') as f: 490 f.write('ltp_packages := \\\n ') 491 f.write(' \\\n '.join(sorted(self._packages))) 492 self._packages = [] 493 494 def ParseAll(self, ltp_root): 495 '''Parse outputs from both 'make' and 'make install'. 496 497 Args: 498 ltp_root: string 499 ''' 500 parser = make_parser.MakeParser(ltp_root) 501 self.ParseInput(parser.ParseFile(MAKE_DRY_RUN_FILE_NAME), ltp_root) 502 parser = make_install_parser.MakeInstallParser(ltp_root) 503 self.ParseInput(parser.ParseFile(MAKE_INSTALL_DRY_RUN_FILE_NAME), ltp_root) 504 505 def GetUnusedCustomCFlagsTargets(self): 506 '''Get targets that have custom cflags, but that weren't built.''' 507 return list(self._unused_custom_cflags) 508 509 510def main(): 511 parser = argparse.ArgumentParser( 512 description='Generate Android.mk from parsed LTP make output') 513 parser.add_argument( 514 '--ltp_root', dest='ltp_root', required=True, help='LTP root dir') 515 parser.add_argument( 516 '--output_mk_path', 517 dest='output_mk_path', 518 required=True, 519 help='output makefile path') 520 parser.add_argument( 521 '--output_bp_path', 522 dest='output_bp_path', 523 required=True, 524 help='output blueprint path') 525 parser.add_argument( 526 '--output_plist_path', 527 required=True, 528 help='output package list path') 529 parser.add_argument( 530 '--custom_cflags_file', 531 dest='custom_cflags_file', 532 required=True, 533 help='file with custom per-module cflags. empty means no file.') 534 args = parser.parse_args() 535 536 custom_cflags = {} 537 if args.custom_cflags_file: 538 # The file is expected to just be a JSON map of string -> [string], e.g. 539 # {"testcases/kernel/syscalls/getcwd/getcwd02": ["-DFOO", "-O3"]} 540 with open(args.custom_cflags_file) as f: 541 custom_cflags = json.load(f) 542 543 generator = BuildGenerator(custom_cflags) 544 generator.ParseAll(args.ltp_root) 545 generator.WriteAndroidMk(args.output_mk_path) 546 generator.WriteAndroidBp(args.output_bp_path) 547 generator.WritePackageList(args.output_plist_path) 548 549 unused_cflags_targs = generator.GetUnusedCustomCFlagsTargets() 550 if unused_cflags_targs: 551 print 'NOTE: Tests had custom cflags, but were never seen: {}'.format( 552 ', '.join(unused_cflags_targs)) 553 554 print 'Finished!' 555 556 557if __name__ == '__main__': 558 main() 559