1#!/usr/bin/env python 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import argparse 18import datetime 19import logging 20import json 21import os 22import shutil 23import subprocess 24import sys 25import webbrowser 26 27from run_host_unit_tests import * 28""" 29This script is used to generate code coverage results host supported libraries. 30The script by default will generate an html report that summarizes the coverage 31results of the specified tests. The results can also be browsed to provide a 32report of which lines have been traveled upon execution of the binary. 33 34NOTE: Code that is compiled out or hidden by a #DEFINE will be listed as 35having been executed 0 times, thus reducing overall coverage. 36 37The steps in order to add coverage support to a new library and its 38corrisponding host test are as follows. 39 401. Add "clang_file_coverage" (defined in //build/Android.bp) as a default to the 41 source library(s) you want statistics for. 42 NOTE: Forgoing this default will cause no coverage data to be generated for 43 the source files in the library. 44 452. Add "clang_coverage_bin" as a default to the host supported test binary that 46 excercises the libraries that you covered in step 1. 47 NOTE: Forgetting to add this will cause there to be *NO* coverage data 48 generated when the binary is run. 49 503. Add the host test binary name and the files/directories you want coverage 51 statistics for to the COVERAGE_TESTS variable defined below. You may add 52 individual filenames or a directory to be tested. 53 NOTE: Avoid using a / at the beginning of a covered_files entry as this 54 breaks how the coverage generator resolves filenames. 55 56TODO: Support generating XML data and printing results to standard out. 57""" 58 59COVERAGE_TESTS = [ 60 { 61 "test_name": "net_test_avrcp", 62 "covered_files": [ 63 "system/bt/profile/avrcp", 64 ], 65 }, 66 { 67 "test_name": "bluetooth_test_sdp", 68 "covered_files": [ 69 "system/bt/profile/sdp", 70 ], 71 }, 72 { 73 "test_name": 74 "test-vendor_test_host", 75 "covered_files": [ 76 "system/bt/vendor_libs/test_vendor_lib/include", 77 "system/bt/vendor_libs/test_vendor_lib/src", 78 ], 79 }, 80 { 81 "test_name": "rootcanal-packets_test_host", 82 "covered_files": [ 83 "system/bt/vendor_libs/test_vendor_lib/packets", 84 ], 85 }, 86 { 87 "test_name": "bluetooth_test_common", 88 "covered_files": [ 89 "system/bt/common", 90 ], 91 }, 92] 93 94WORKING_DIR = '/tmp/coverage' 95SOONG_UI_BASH = 'build/soong/soong_ui.bash' 96LLVM_DIR = 'prebuilts/clang/host/linux-x86/clang-r353983b/bin' 97LLVM_MERGE = LLVM_DIR + '/llvm-profdata' 98LLVM_COV = LLVM_DIR + '/llvm-cov' 99 100 101def write_root_html_head(f): 102 # Write the header part of the root html file. This was pulled from the 103 # page source of one of the generated html files. 104 f.write("<!doctype html><html><head>" \ 105 "<meta name='viewport' content='width=device-width,initial-scale=1'><met" \ 106 "a charset='UTF-8'><link rel='stylesheet' type='text/css' href='style.cs" \ 107 "s'></head><body><h2>Coverage Report</h2><h4>Created: " + 108 str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M')) + 109 "</h4><p>Click <a href='http://clang.llvm.org/docs/SourceBasedCodeCovera" \ 110 "ge.html#interpreting-reports'>here</a> for information about interpreti" \ 111 "ng this report.</p><div class='centered'><table><tr><td class='column-e" \ 112 "ntry-bold'>Filename</td><td class='column-entry-bold'>Function Coverage" \ 113 "</td><td class='column-entry-bold'>Instantiation Coverage</td><td class" \ 114 "='column-entry-bold'>Line Coverage</td><td class='column-entry-bold'>Re" \ 115 "gion Coverage</td></tr>" 116 ) 117 118 119def write_root_html_column(f, covered, count): 120 percent = covered * 100.0 / count 121 value = "%.2f%% (%d/%d) " % (percent, covered, count) 122 color = 'column-entry-yellow' 123 if percent == 100: 124 color = 'column-entry-green' 125 if percent < 80.0: 126 color = 'column-entry-red' 127 f.write("<td class=\'" + color + "\'><pre>" + value + "</pre></td>") 128 129 130def write_root_html_rows(f, tests): 131 totals = { 132 "functions": { 133 "covered": 0, 134 "count": 0 135 }, 136 "instantiations": { 137 "covered": 0, 138 "count": 0 139 }, 140 "lines": { 141 "covered": 0, 142 "count": 0 143 }, 144 "regions": { 145 "covered": 0, 146 "count": 0 147 } 148 } 149 150 # Write the tests with their coverage summaries. 151 for test in tests: 152 test_name = test['test_name'] 153 covered_files = test['covered_files'] 154 json_results = generate_coverage_json(test) 155 test_totals = json_results['data'][0]['totals'] 156 157 f.write("<tr class='light-row'><td><pre><a href=\'" + 158 os.path.join(test_name, "index.html") + "\'>" + test_name + 159 "</a></pre></td>") 160 for field_name in ['functions', 'instantiations', 'lines', 'regions']: 161 field = test_totals[field_name] 162 totals[field_name]['covered'] += field['covered'] 163 totals[field_name]['count'] += field['count'] 164 write_root_html_column(f, field['covered'], field['count']) 165 f.write("</tr>") 166 167 #Write the totals row. 168 f.write("<tr class='light-row-bold'><td><pre>Totals</a></pre></td>") 169 for field_name in ['functions', 'instantiations', 'lines', 'regions']: 170 field = totals[field_name] 171 write_root_html_column(f, field['covered'], field['count']) 172 f.write("</tr>") 173 174 175def write_root_html_tail(f): 176 # Pulled from the generated html coverage report. 177 f.write("</table></div><h5>Generated by llvm-cov -- llvm version 7.0.2svn<" \ 178 "/h5></body></html>") 179 180 181def generate_root_html(tests): 182 # Copy the css file from one of the coverage reports. 183 source_file = os.path.join( 184 os.path.join(WORKING_DIR, tests[0]['test_name']), "style.css") 185 dest_file = os.path.join(WORKING_DIR, "style.css") 186 shutil.copy2(source_file, dest_file) 187 188 # Write the root index.html file that sumarizes all the tests. 189 f = open(os.path.join(WORKING_DIR, "index.html"), "w") 190 write_root_html_head(f) 191 write_root_html_rows(f, tests) 192 write_root_html_tail(f) 193 194 195def get_profraw_for_test(test_name): 196 test_root = get_native_test_root_or_die() 197 test_cmd = os.path.join(os.path.join(test_root, test_name), test_name) 198 if not os.path.isfile(test_cmd): 199 logging.error('The test ' + test_name + 200 ' does not exist, please compile first') 201 sys.exit(1) 202 203 profraw_file_name = test_name + ".profraw" 204 profraw_path = os.path.join(WORKING_DIR, 205 os.path.join(test_name, profraw_file_name)) 206 llvm_env_var = "LLVM_PROFILE_FILE=\"" + profraw_path + "\"" 207 208 test_cmd = llvm_env_var + " " + test_cmd 209 logging.info('Generating profraw data for ' + test_name) 210 logging.debug('cmd: ' + test_cmd) 211 if subprocess.call(test_cmd, shell=True) != 0: 212 logging.error( 213 'Test ' + test_name + 214 ' failed. Please fix the test before generating coverage.') 215 sys.exit(1) 216 217 if not os.path.isfile(profraw_path): 218 logging.error( 219 'Generating the profraw file failed. Did you remember to add the proper compiler flags to your build?' 220 ) 221 sys.exit(1) 222 223 return profraw_file_name 224 225 226def merge_profraw_data(test_name): 227 cmd = [] 228 cmd.append(os.path.join(get_android_root_or_die(), LLVM_MERGE + " merge ")) 229 230 test_working_dir = os.path.join(WORKING_DIR, test_name) 231 cmd.append(os.path.join(test_working_dir, test_name + ".profraw")) 232 profdata_file = os.path.join(test_working_dir, test_name + ".profdata") 233 234 cmd.append('-o ' + profdata_file) 235 logging.info('Combining profraw files into profdata for ' + test_name) 236 logging.debug('cmd: ' + " ".join(cmd)) 237 if subprocess.call(" ".join(cmd), shell=True) != 0: 238 logging.error('Failed to merge profraw files for ' + test_name) 239 sys.exit(1) 240 241 242def generate_coverage_html(test): 243 COVERAGE_ROOT = '/proc/self/cwd' 244 245 test_name = test['test_name'] 246 file_list = test['covered_files'] 247 248 test_working_dir = os.path.join(WORKING_DIR, test_name) 249 test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata") 250 251 cmd = [ 252 os.path.join(get_android_root_or_die(), LLVM_COV), "show", 253 "-format=html", "-summary-only", "-show-line-counts-or-regions", 254 "-show-instantiation-summary", "-instr-profile=" + test_profdata_file, 255 "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" + 256 get_android_root_or_die() + "\"", "-output-dir=" + test_working_dir 257 ] 258 259 # We have to have one object file not as an argument otherwise we can't specify source files. 260 test_cmd = os.path.join( 261 os.path.join(get_native_test_root_or_die(), test_name), test_name) 262 cmd.append(test_cmd) 263 264 # Filter out the specific files we want coverage for 265 for filename in file_list: 266 cmd.append(os.path.join(get_android_root_or_die(), filename)) 267 268 logging.info('Generating coverage report for ' + test['test_name']) 269 logging.debug('cmd: ' + " ".join(cmd)) 270 if subprocess.call(" ".join(cmd), shell=True) != 0: 271 logging.error('Failed to generate coverage for ' + test['test_name']) 272 sys.exit(1) 273 274 275def generate_coverage_json(test): 276 COVERAGE_ROOT = '/proc/self/cwd' 277 test_name = test['test_name'] 278 file_list = test['covered_files'] 279 280 test_working_dir = os.path.join(WORKING_DIR, test_name) 281 test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata") 282 283 cmd = [ 284 os.path.join(get_android_root_or_die(), LLVM_COV), 285 "export", 286 "-summary-only", 287 "-show-region-summary", 288 "-instr-profile=" + test_profdata_file, 289 "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" + 290 get_android_root_or_die() + "\"", 291 ] 292 293 test_cmd = os.path.join( 294 os.path.join(get_native_test_root_or_die(), test_name), test_name) 295 cmd.append(test_cmd) 296 297 # Filter out the specific files we want coverage for 298 for filename in file_list: 299 cmd.append(os.path.join(get_android_root_or_die(), filename)) 300 301 logging.info('Generating coverage json for ' + test['test_name']) 302 logging.debug('cmd: ' + " ".join(cmd)) 303 304 json_str = subprocess.check_output(" ".join(cmd), shell=True) 305 return json.loads(json_str) 306 307 308def write_json_summary(test): 309 test_name = test['test_name'] 310 test_working_dir = os.path.join(WORKING_DIR, test_name) 311 test_json_summary_file = os.path.join(test_working_dir, test_name + '.json') 312 logging.debug('Writing json summary file: ' + test_json_summary_file) 313 json_file = open(test_json_summary_file, 'w') 314 json.dump(generate_coverage_json(test), json_file) 315 json_file.close() 316 317 318def list_tests(): 319 for test in COVERAGE_TESTS: 320 print "Test Name: " + test['test_name'] 321 print "Covered Files: " 322 for covered_file in test['covered_files']: 323 print " " + covered_file 324 print 325 326 327def main(): 328 parser = argparse.ArgumentParser( 329 description='Generate code coverage for enabled tests.') 330 parser.add_argument( 331 '-l', 332 '--list-tests', 333 action='store_true', 334 dest='list_tests', 335 help='List all the available tests to be run as well as covered files.') 336 parser.add_argument( 337 '-a', '--all', 338 action='store_true', 339 help='Runs all available tests and prints their outputs. If no tests ' \ 340 'are specified via the -t option all tests will be run.') 341 parser.add_argument( 342 '-t', '--test', 343 dest='tests', 344 action='append', 345 type=str, 346 metavar='TESTNAME', 347 default=[], 348 help='Specifies a test to be run. Multiple tests can be specified by ' \ 349 'using this option multiple times. ' \ 350 'Example: \"gen_coverage.py -t test1 -t test2\"') 351 parser.add_argument( 352 '-o', '--output', 353 type=str, 354 metavar='DIRECTORY', 355 default='/tmp/coverage', 356 help='Specifies the directory to store all files. The directory will be ' \ 357 'created if it does not exist. Default is \"/tmp/coverage\"') 358 parser.add_argument( 359 '-s', 360 '--skip-html', 361 dest='skip_html', 362 action='store_true', 363 help='Skip opening up the results of the coverage report in a browser.') 364 parser.add_argument( 365 '-j', 366 '--json-file', 367 dest='json_file', 368 action='store_true', 369 help='Write out summary results to json file in test directory.') 370 371 logging.basicConfig( 372 stream=sys.stderr, 373 level=logging.DEBUG, 374 format='%(levelname)s %(message)s') 375 logging.addLevelName( 376 logging.DEBUG, 377 "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.DEBUG)) 378 logging.addLevelName( 379 logging.INFO, 380 "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.INFO)) 381 logging.addLevelName( 382 logging.WARNING, 383 "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.WARNING)) 384 logging.addLevelName( 385 logging.ERROR, 386 "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.ERROR)) 387 388 args = parser.parse_args() 389 logging.debug("Args: " + str(args)) 390 391 # Set the working directory 392 global WORKING_DIR 393 WORKING_DIR = os.path.abspath(args.output) 394 logging.debug("Working Dir: " + WORKING_DIR) 395 396 # Print out the list of tests then exit 397 if args.list_tests: 398 list_tests() 399 sys.exit(0) 400 401 # Check to see if a test was specified and if so only generate coverage for 402 # that test. 403 if len(args.tests) == 0: 404 args.all = True 405 406 tests_to_run = [] 407 for test in COVERAGE_TESTS: 408 if args.all or test['test_name'] in args.tests: 409 tests_to_run.append(test) 410 if test['test_name'] in args.tests: 411 args.tests.remove(test['test_name']) 412 413 # Error if a test was specified but doesn't exist. 414 if len(args.tests) != 0: 415 for test_name in args.tests: 416 logging.error('\"' + test_name + 417 '\" was not found in the list of available tests.') 418 sys.exit(1) 419 420 # Generate the info for the tests 421 for test in tests_to_run: 422 logging.info('Getting coverage for ' + test['test_name']) 423 get_profraw_for_test(test['test_name']) 424 merge_profraw_data(test['test_name']) 425 if args.json_file: 426 write_json_summary(test) 427 generate_coverage_html(test) 428 429 # Generate the root index.html page that sumarizes all of the coverage reports. 430 generate_root_html(tests_to_run) 431 432 # Open the results in a browser. 433 if not args.skip_html: 434 webbrowser.open('file://' + os.path.join(WORKING_DIR, 'index.html')) 435 436 437if __name__ == '__main__': 438 main() 439