1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18# Parses the output of parse_ltp_{make,make_install} and generates a 19# corresponding Android.mk. 20# 21# This process is split into two steps so this second step can later be replaced 22# with an Android.bp generator. 23 24import argparse 25import fileinput 26import json 27import os 28import re 29 30import make_parser 31import make_install_parser 32 33MAKE_DRY_RUN_FILE_NAME = os.path.join('dump', 'make_dry_run.dump') 34MAKE_INSTALL_DRY_RUN_FILE_NAME = os.path.join('dump', 'make_install_dry_run.dump') 35DISABLED_TESTS_FILE_NAME = 'disabled_tests.txt' 36DISABLED_LIBS_FILE_NAME = 'disabled_libs.txt' 37DISABLED_CFLAGS_FILE_NAME = 'disabled_cflags.txt' 38 39 40class BuildGenerator(object): 41 '''A class to parse make output and convert the result to Android.ltp.mk modules. 42 43 Attributes: 44 _bp_result: directory of list of strings for blueprint file keyed by target name 45 _prebuilt_bp_result: directory of list of strings for blueprint keyed by target 46 name 47 _custom_cflags: dict of string (module name) to lists of strings (cflags 48 to add for said module) 49 _unused_custom_cflags: set of strings; tracks the modules with custom 50 cflags that we haven't yet seen 51 _packages: list of strings of packages for package list file 52 ''' 53 54 def __init__(self, custom_cflags): 55 self._bp_result = {} 56 self._prebuilt_bp_result = {} 57 self._custom_cflags = custom_cflags 58 self._unused_custom_cflags = set(custom_cflags) 59 self._packages = [] 60 61 def UniqueKeepOrder(self, sequence): 62 '''Get a copy of list where items are unique and order is preserved. 63 64 Args: 65 sequence: a sequence, can be a list, tuple, or other iterable 66 67 Returns: 68 a list where items copied from input sequence are unique 69 and order is preserved. 70 ''' 71 seen = set() 72 return [x for x in sequence if not (x in seen or seen.add(x))] 73 74 def ReadCommentedText(self, file_path): 75 '''Read pound commented text file into a list of lines. 76 77 Comments or empty lines will be excluded 78 79 Args: 80 file_path: string 81 ''' 82 ret = set() 83 with open(file_path, 'r') as f: 84 lines = [line.strip() for line in f.readlines()] 85 ret = set([s for s in lines if s and not s.startswith('#')]) 86 87 return ret 88 89 def ArTargetToLibraryName(self, ar_target): 90 '''Convert ar target to library name. 91 92 Args: 93 ar_target: string 94 ''' 95 return os.path.basename(ar_target)[len('lib'):-len('.a')] 96 97 def BuildExecutable(self, cc_target, local_src_files, local_cflags, 98 local_c_includes, local_libraries, ltp_libs, 99 ltp_libs_used, ltp_names_used): 100 '''Build a test module. 101 102 Args: 103 cc_target: string 104 local_src_files: list of string 105 local_cflags: list of string 106 local_c_includes: list of string 107 local_libraries: list of string 108 ltp_libs: list of string 109 ltp_libs_used: set of string 110 ltp_names_used: set of string, set of already used cc_target basenames 111 ''' 112 base_name = os.path.basename(cc_target) 113 if base_name in ltp_names_used: 114 print('ERROR: base name {} of cc_target {} already used. Skipping...'.format( 115 base_name, cc_target)) 116 return 117 ltp_names_used.add(base_name) 118 119 if cc_target in self._custom_cflags: 120 local_cflags.extend(self._custom_cflags[cc_target]) 121 self._unused_custom_cflags.remove(cc_target) 122 123 # ltp_defaults already adds the include directory 124 local_c_includes = [i for i in local_c_includes if i != 'include'] 125 target_name = 'ltp_%s' % base_name 126 target_bp = [] 127 128 self._packages.append(target_name) 129 130 target_bp.append('cc_test {') 131 target_bp.append(' name: "%s",' % target_name) 132 target_bp.append(' stem: "%s",' % base_name) 133 target_bp.append(' defaults: ["ltp_test_defaults"],') 134 135 if len(local_src_files) == 1: 136 target_bp.append(' srcs: ["%s"],' % list(local_src_files)[0]) 137 else: 138 target_bp.append(' srcs: [') 139 for src in sorted(local_src_files): 140 target_bp.append(' "%s",' % src) 141 target_bp.append(' ],') 142 143 if len(local_cflags) == 1: 144 target_bp.append(' cflags: ["%s"],' % list(local_cflags)[0]) 145 elif len(local_cflags) > 1: 146 target_bp.append(' cflags: [') 147 for cflag in sorted(local_cflags): 148 target_bp.append(' "%s",' % cflag) 149 target_bp.append(' ],') 150 151 if len(local_c_includes) == 1: 152 target_bp.append(' local_include_dirs: ["%s"],' % list(local_c_includes)[0]) 153 elif len(local_c_includes) > 1: 154 target_bp.append(' local_include_dirs: [') 155 for d in sorted(local_c_includes): 156 target_bp.append(' "%s",' % d) 157 target_bp.append(' ],') 158 159 bionic_builtin_libs = set(['m', 'rt', 'pthread', 'util']) 160 filtered_libs = set(local_libraries).difference(bionic_builtin_libs) 161 162 static_libraries = set(i for i in local_libraries if i in ltp_libs) 163 if len(static_libraries) == 1: 164 target_bp.append(' static_libs: ["libltp_%s"],' % list(static_libraries)[0]) 165 elif len(static_libraries) > 1: 166 target_bp.append(' static_libs: [') 167 for lib in sorted(static_libraries): 168 target_bp.append(' "libltp_%s",' % lib) 169 target_bp.append(' ],') 170 171 for lib in static_libraries: 172 ltp_libs_used.add(lib) 173 174 shared_libraries = set(i for i in filtered_libs if i not in ltp_libs) 175 if len(shared_libraries) == 1: 176 target_bp.append(' shared_libs: ["lib%s"],' % list(shared_libraries)[0]) 177 elif len(shared_libraries) > 1: 178 target_bp.append(' shared_libs: [') 179 for lib in sorted(shared_libraries): 180 target_bp.append(' "lib%s",' % lib) 181 target_bp.append(' ],') 182 183 target_bp.append('}') 184 target_bp.append('') 185 self._bp_result[target_name] = target_bp 186 187 def BuildStaticLibrary(self, ar_target, local_src_files, local_cflags, 188 local_c_includes): 189 '''Build a library module. 190 191 Args: 192 ar_target: string 193 local_src_files: list of string 194 local_cflags: list of string 195 local_c_includes: list of string 196 ''' 197 target_name = 'libltp_%s' % self.ArTargetToLibraryName(ar_target) 198 target_bp = [] 199 target_bp.append('cc_library_static {') 200 target_bp.append(' name: "%s",' % target_name) 201 target_bp.append(' defaults: ["ltp_defaults"],') 202 203 if len(local_c_includes): 204 target_bp.append(' local_include_dirs: [') 205 for d in local_c_includes: 206 target_bp.append(' "%s",' % d) 207 target_bp.append(' ],') 208 209 if len(local_cflags): 210 target_bp.append(' cflags: [') 211 for cflag in local_cflags: 212 target_bp.append(' "%s",' % cflag) 213 target_bp.append(' ],') 214 215 target_bp.append(' srcs: [') 216 for src in local_src_files: 217 target_bp.append(' "%s",' % src) 218 target_bp.append(' ],') 219 220 target_bp.append('}') 221 target_bp.append('') 222 self._bp_result[target_name] = target_bp 223 224 def BuildShellScript(self, install_target, local_src_file): 225 '''Build a shell script. 226 227 Args: 228 install_target: string 229 local_src_file: string 230 ''' 231 base_name = os.path.basename(install_target) 232 bp_result = [] 233 234 module = 'ltp_%s' % install_target.replace('/', '_') 235 self._packages.append(module) 236 237 module_dir = os.path.dirname(install_target) 238 module_stem = os.path.basename(install_target) 239 240 bp_result.append('sh_test {') 241 bp_result.append(' name: "%s",' % module) 242 bp_result.append(' src: "%s",' % local_src_file) 243 bp_result.append(' sub_dir: "ltp/%s",' % module_dir) 244 bp_result.append(' filename: "%s",' % module_stem) 245 bp_result.append(' compile_multilib: "both",') 246 bp_result.append('}') 247 bp_result.append('') 248 249 self._bp_result[module] = bp_result 250 251 def BuildPrebuiltBp(self, install_target, local_src_file): 252 '''Build a prebuild module for using Android.bp. 253 254 Args: 255 install_target: string 256 local_src_file: string 257 ''' 258 base_name = os.path.basename(install_target) 259 # The original local_src_file is from external/ltp, but for bp the root 260 # will be external/ltp/testcases. 261 src = local_src_file.replace('testcases/', '', 1) 262 module = 'ltp_%s' % install_target.replace('/', '_') 263 module_dir = os.path.dirname(install_target) 264 module_stem = os.path.basename(install_target) 265 266 bp_result = [] 267 bp_result.append('sh_test {') 268 bp_result.append(' name: "%s",' % module) 269 bp_result.append(' src: "%s",' % src) 270 bp_result.append(' sub_dir: "ltp/%s",' % module_dir) 271 bp_result.append(' filename: "%s",' % module_stem) 272 bp_result.append(' compile_multilib: "both",') 273 bp_result.append(' auto_gen_config: false,') 274 bp_result.append('}') 275 bp_result.append('') 276 277 self._prebuilt_bp_result[base_name] = bp_result 278 self._packages.append(module) 279 280 def HandleParsedRule(self, line, rules): 281 '''Prepare parse rules. 282 283 Args: 284 line: string 285 rules: dictionary {string, dictionary} 286 ''' 287 groups = re.match(r'(.*)\[\'(.*)\'\] = \[(.*)\]', line).groups() 288 rule = groups[0] 289 rule_key = groups[1] 290 if groups[2] == '': 291 rule_value = [] 292 else: 293 rule_value = list(i.strip()[1:-1] for i in groups[2].split(',')) 294 295 rule_value = self.UniqueKeepOrder(rule_value) 296 rules.setdefault(rule, {})[rule_key] = rule_value 297 298 def ParseInput(self, input_list, ltp_root): 299 '''Parse a interpreted make output and produce Android.ltp.mk module. 300 301 Args: 302 input_list: list of string 303 ''' 304 disabled_tests = self.ReadCommentedText(DISABLED_TESTS_FILE_NAME) 305 disabled_libs = self.ReadCommentedText(DISABLED_LIBS_FILE_NAME) 306 disabled_cflags = self.ReadCommentedText(DISABLED_CFLAGS_FILE_NAME) 307 308 rules = {} 309 for line in input_list: 310 self.HandleParsedRule(line.strip(), rules) 311 312 # .a target -> .o files 313 ar = rules.get('ar', {}) 314 # executable target -> .o files 315 cc_link = rules.get('cc_link', {}) 316 # .o target -> .c file 317 cc_compile = rules.get('cc_compile', {}) 318 # executable target -> .c files 319 cc_compilelink = rules.get('cc_compilelink', {}) 320 # Target name -> CFLAGS passed to gcc 321 cc_flags = rules.get('cc_flags', {}) 322 # Target name -> -I paths passed to gcc 323 cc_includes = rules.get('cc_includes', {}) 324 # Target name -> -l paths passed to gcc 325 cc_libraries = rules.get('cc_libraries', {}) 326 # target -> prebuilt source 327 install = rules.get('install', {}) 328 329 # All libraries used by any LTP test (built or not) 330 ltp_libs = set(self.ArTargetToLibraryName(i) for i in ar.keys()) 331 # All libraries used by the LTP tests we actually build 332 ltp_libs_used = set() 333 ltp_names_used = set() 334 335 # Remove -Wno-error from cflags, we don't want to print warnings. 336 # Silence individual warnings in ltp_defaults or fix them. 337 for target in cc_flags: 338 if '-Wno-error' in cc_flags[target]: 339 cc_flags[target].remove('-Wno-error') 340 341 print( 342 "Disabled lib tests: Test cases listed here are" 343 "suggested to be disabled since they require a disabled library. " 344 "Please copy and paste them into disabled_tests.txt\n") 345 for i in cc_libraries: 346 if len(set(cc_libraries[i]).intersection(disabled_libs)) > 0: 347 if not os.path.basename(i) in disabled_tests: 348 print(os.path.basename(i)) 349 350 print("Disabled_cflag tests: Test cases listed here are" 351 "suggested to be disabled since they require a disabled cflag. " 352 "Please copy and paste them into disabled_tests.txt\n") 353 for i in cc_flags: 354 if len(set(cc_flags[i]).intersection(disabled_cflags)) > 0: 355 module_name = os.path.basename(i) 356 idx = module_name.find('_') 357 if idx > 0: 358 module_name = module_name[:idx] 359 print(module_name) 360 361 # Remove include directories that don't exist. They're an error in 362 # Soong. 363 for target in cc_includes: 364 cc_includes[target] = [i for i in cc_includes[target] if os.path.isdir(os.path.join(ltp_root, i))] 365 366 for target in cc_compilelink: 367 module_name = os.path.basename(target) 368 if module_name in disabled_tests: 369 continue 370 local_src_files = [] 371 src_files = cc_compilelink[target] 372 for i in src_files: 373 # some targets may have a mix of .c and .o files in srcs 374 # find the .c files to build those .o from cc_compile targets 375 if i.endswith('.o'): 376 if i not in cc_compile: 377 raise Exception("Not found: %s when trying to compile target %s" % (i, target)) 378 local_src_files.extend(cc_compile[i]) 379 else: 380 local_src_files.append(i) 381 local_cflags = cc_flags[target] 382 local_c_includes = cc_includes[target] 383 local_libraries = cc_libraries[target] 384 if len(set(local_libraries).intersection(disabled_libs)) > 0: 385 continue 386 if len(set(local_cflags).intersection(disabled_cflags)) > 0: 387 continue 388 self.BuildExecutable(target, local_src_files, local_cflags, 389 local_c_includes, local_libraries, ltp_libs, 390 ltp_libs_used, ltp_names_used) 391 392 for target in cc_link: 393 if os.path.basename(target) in disabled_tests: 394 continue 395 local_src_files = set() 396 local_cflags = set() 397 local_c_includes = set() 398 local_libraries = cc_libraries[target] 399 # Accumulate flags for all .c files needed to build the .o files. 400 # (Android.mk requires a consistent set of flags across a given target. 401 # Thankfully using the superset of all flags in the target works fine 402 # with LTP tests.) 403 for obj in cc_link[target]: 404 for i in cc_compile[obj]: 405 local_src_files.add(i) 406 for i in cc_flags[obj]: 407 local_cflags.add(i) 408 for i in cc_includes[obj]: 409 local_c_includes.add(i) 410 if len(set(local_libraries).intersection(disabled_libs)) > 0: 411 continue 412 if len(set(local_cflags).intersection(disabled_cflags)) > 0: 413 continue 414 415 self.BuildExecutable(target, local_src_files, local_cflags, 416 local_c_includes, local_libraries, ltp_libs, 417 ltp_libs_used, ltp_names_used) 418 419 for target in ar: 420 # Disabled ltp library is already excluded 421 # since it won't be in ltp_libs_used 422 if not self.ArTargetToLibraryName(target) in ltp_libs_used: 423 continue 424 425 local_src_files = set() 426 local_cflags = set() 427 local_c_includes = set() 428 429 # TODO: disabled cflags 430 431 for obj in ar[target]: 432 for i in cc_compile[obj]: 433 local_src_files.add(i) 434 for i in cc_flags[obj]: 435 local_cflags.add(i) 436 for i in cc_includes[obj]: 437 local_c_includes.add(i) 438 439 if len(set(local_cflags).intersection(disabled_cflags)) > 0: 440 continue 441 442 local_src_files = sorted(local_src_files) 443 local_cflags = sorted(local_cflags) 444 local_c_includes = sorted(local_c_includes) 445 446 self.BuildStaticLibrary(target, local_src_files, local_cflags, 447 local_c_includes) 448 449 for target in install: 450 # Check if the absolute path to the prebuilt (relative to LTP_ROOT) 451 # is disabled. This is helpful in case there are duplicates with basename 452 # of the prebuilt. 453 # e.g. 454 # ./ testcases / kernel / fs / fs_bind / move / test01 455 # ./ testcases / kernel / fs / fs_bind / cloneNS / test01 456 # ./ testcases / kernel / fs / fs_bind / regression / test01 457 # ./ testcases / kernel / fs / fs_bind / rbind / test01 458 # ./ testcases / kernel / fs / fs_bind / bind / test01 459 if target in disabled_tests: 460 continue 461 if os.path.basename(target) in disabled_tests: 462 continue 463 local_src_files = install[target] 464 assert len(local_src_files) == 1 465 466 if target.startswith("testcases/bin/"): 467 self.BuildShellScript(target, local_src_files[0]) 468 else: 469 self.BuildPrebuiltBp(target, local_src_files[0]) 470 471 def WriteAndroidBp(self, output_path): 472 '''Write parse result to blueprint file. 473 474 Args: 475 output_path: string 476 ''' 477 with open(output_path, 'a') as f: 478 for k in sorted(self._bp_result.keys()): 479 f.write('\n'.join(self._bp_result[k])) 480 f.write('\n') 481 self._bp_result = {} 482 483 def WritePrebuiltAndroidBp(self, output_path): 484 '''Write parse result to blueprint file. 485 486 Args: 487 output_path: string 488 ''' 489 with open(output_path, 'a') as f: 490 f.write('package {\n') 491 f.write(' default_applicable_licenses: ["external_ltp_license"],\n') 492 f.write('}\n\n') 493 for k in sorted(self._prebuilt_bp_result.keys()): 494 f.write('\n'.join(self._prebuilt_bp_result[k])) 495 f.write('\n') 496 self._prebuilt_bp_result = {} 497 498 def WriteLtpMainAndroidBp(self, output_path): 499 '''Write the blueprint file of ltp main module. 500 501 Args: 502 output_path: string 503 ''' 504 bp_result = [] 505 bp_result.append('package {') 506 bp_result.append(' default_applicable_licenses: ["external_ltp_license"],') 507 bp_result.append('}') 508 bp_result.append('') 509 bp_result.append('sh_test {') 510 bp_result.append(' name: "ltp",') 511 bp_result.append(' src: "tools/disabled_tests.txt",') 512 bp_result.append(' sub_dir: "ltp",') 513 bp_result.append(' data: [":ltp_runtests"],') 514 bp_result.append(' filename_from_src: true,') 515 bp_result.append(' compile_multilib: "both",') 516 bp_result.append(' auto_gen_config: false,') 517 bp_result.append(' required: [') 518 for package in sorted(self._packages): 519 bp_result.append(' "%s",' % package) 520 bp_result.append(' ],') 521 bp_result.append('}') 522 523 with open(output_path, 'a') as f: 524 f.write('\n'.join(bp_result)) 525 f.write('\n') 526 527 def ParseAll(self, ltp_root): 528 '''Parse outputs from both 'make' and 'make install'. 529 530 Args: 531 ltp_root: string 532 ''' 533 parser = make_parser.MakeParser(ltp_root) 534 self.ParseInput(parser.ParseFile(MAKE_DRY_RUN_FILE_NAME), ltp_root) 535 parser = make_install_parser.MakeInstallParser(ltp_root) 536 self.ParseInput(parser.ParseFile(MAKE_INSTALL_DRY_RUN_FILE_NAME), ltp_root) 537 538 def GetUnusedCustomCFlagsTargets(self): 539 '''Get targets that have custom cflags, but that weren't built.''' 540 return list(self._unused_custom_cflags) 541 542 543def main(): 544 parser = argparse.ArgumentParser( 545 description='Generate Android.mk from parsed LTP make output') 546 parser.add_argument( 547 '--ltp_root', dest='ltp_root', required=True, help='LTP root dir') 548 parser.add_argument( 549 '--output_prebuilt_ltp_testcase_bp_path', 550 dest='output_prebuilt_ltp_testcase_bp_path', 551 required=True, 552 help='output prebuilt test case blueprint path') 553 parser.add_argument( 554 '--output_ltp_main_bp_path', 555 dest='output_ltp_main_bp_path', 556 required=True, 557 help='output ltp main blueprint path') 558 parser.add_argument( 559 '--output_bp_path', 560 dest='output_bp_path', 561 required=True, 562 help='output blueprint path') 563 parser.add_argument( 564 '--custom_cflags_file', 565 dest='custom_cflags_file', 566 required=True, 567 help='file with custom per-module cflags. empty means no file.') 568 args = parser.parse_args() 569 570 custom_cflags = {} 571 if args.custom_cflags_file: 572 # The file is expected to just be a JSON map of string -> [string], e.g. 573 # {"testcases/kernel/syscalls/getcwd/getcwd02": ["-DFOO", "-O3"]} 574 with open(args.custom_cflags_file) as f: 575 custom_cflags = json.load(f) 576 577 generator = BuildGenerator(custom_cflags) 578 generator.ParseAll(args.ltp_root) 579 generator.WritePrebuiltAndroidBp(args.output_prebuilt_ltp_testcase_bp_path) 580 generator.WriteLtpMainAndroidBp(args.output_ltp_main_bp_path) 581 generator.WriteAndroidBp(args.output_bp_path) 582 583 unused_cflags_targs = generator.GetUnusedCustomCFlagsTargets() 584 if unused_cflags_targs: 585 print('NOTE: Tests had custom cflags, but were never seen: {}'.format( 586 ', '.join(unused_cflags_targs))) 587 588 print('Finished!') 589 590 591if __name__ == '__main__': 592 main() 593