1#!/usr/bin/env python 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import argparse 18import datetime 19import logging 20import json 21import os 22import shutil 23import subprocess 24import sys 25import webbrowser 26 27from run_host_unit_tests import * 28 29""" 30This script is used to generate code coverage results host supported libraries. 31The script by default will generate an html report that summarizes the coverage 32results of the specified tests. The results can also be browsed to provide a 33report of which lines have been traveled upon execution of the binary. 34 35NOTE: Code that is compiled out or hidden by a #DEFINE will be listed as 36having been executed 0 times, thus reducing overall coverage. 37 38The steps in order to add coverage support to a new library and its 39corrisponding host test are as follows. 40 411. Add "clang_file_coverage" (defined in //build/Android.bp) as a default to the 42 source library(s) you want statistics for. 43 NOTE: Forgoing this default will cause no coverage data to be generated for 44 the source files in the library. 45 462. Add "clang_coverage_bin" as a default to the host supported test binary that 47 excercises the libraries that you covered in step 1. 48 NOTE: Forgetting to add this will cause there to be *NO* coverage data 49 generated when the binary is run. 50 513. Add the host test binary name and the files/directories you want coverage 52 statistics for to the COVERAGE_TESTS variable defined below. You may add 53 individual filenames or a directory to be tested. 54 NOTE: Avoid using a / at the beginning of a covered_files entry as this 55 breaks how the coverage generator resolves filenames. 56 57TODO: Support generating XML data and printing results to standard out. 58""" 59 60COVERAGE_TESTS = [ 61 { 62 "test_name": "net_test_avrcp", 63 "covered_files": [ 64 "system/bt/profile/avrcp", 65 ], 66 }, { 67 "test_name": "bluetooth_test_sdp", 68 "covered_files": [ 69 "system/bt/profile/sdp", 70 ], 71 }, { 72 "test_name": "test-vendor_test_host", 73 "covered_files": [ 74 "system/bt/vendor_libs/test_vendor_lib/include", 75 "system/bt/vendor_libs/test_vendor_lib/src", 76 ], 77 }, { 78 "test_name": "rootcanal-packets_test_host", 79 "covered_files": [ 80 "system/bt/vendor_libs/test_vendor_lib/packets", 81 ], 82 }, { 83 "test_name": "bluetooth_test_common", 84 "covered_files": [ 85 "system/bt/common", 86 ], 87 }, 88] 89 90WORKING_DIR = '/tmp/coverage' 91SOONG_UI_BASH = 'build/soong/soong_ui.bash' 92LLVM_DIR = 'prebuilts/clang/host/linux-x86/clang-r328903/bin' 93LLVM_MERGE = LLVM_DIR + '/llvm-profdata' 94LLVM_COV = LLVM_DIR + '/llvm-cov' 95 96def write_root_html_head(f): 97 # Write the header part of the root html file. This was pulled from the 98 # page source of one of the generated html files. 99 f.write("<!doctype html><html><head>" \ 100 "<meta name='viewport' content='width=device-width,initial-scale=1'><met" \ 101 "a charset='UTF-8'><link rel='stylesheet' type='text/css' href='style.cs" \ 102 "s'></head><body><h2>Coverage Report</h2><h4>Created: " + 103 str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M')) + 104 "</h4><p>Click <a href='http://clang.llvm.org/docs/SourceBasedCodeCovera" \ 105 "ge.html#interpreting-reports'>here</a> for information about interpreti" \ 106 "ng this report.</p><div class='centered'><table><tr><td class='column-e" \ 107 "ntry-bold'>Filename</td><td class='column-entry-bold'>Function Coverage" \ 108 "</td><td class='column-entry-bold'>Instantiation Coverage</td><td class" \ 109 "='column-entry-bold'>Line Coverage</td><td class='column-entry-bold'>Re" \ 110 "gion Coverage</td></tr>" 111 ) 112 113 114def write_root_html_column(f, covered, count): 115 percent = covered * 100.0 / count 116 value = "%.2f%% (%d/%d) " % (percent, covered, count) 117 color = 'column-entry-yellow' 118 if percent == 100: 119 color = 'column-entry-green' 120 if percent < 80.0: 121 color = 'column-entry-red' 122 f.write("<td class=\'" + color + "\'><pre>" + value + "</pre></td>") 123 124 125def write_root_html_rows(f, tests): 126 totals = { 127 "functions":{ 128 "covered": 0, 129 "count": 0 130 }, 131 "instantiations":{ 132 "covered": 0, 133 "count": 0 134 }, 135 "lines":{ 136 "covered": 0, 137 "count": 0 138 }, 139 "regions":{ 140 "covered": 0, 141 "count": 0 142 } 143 } 144 145 # Write the tests with their coverage summaries. 146 for test in tests: 147 test_name = test['test_name'] 148 covered_files = test['covered_files'] 149 json_results = generate_coverage_json(test) 150 test_totals = json_results['data'][0]['totals'] 151 152 f.write("<tr class='light-row'><td><pre><a href=\'" + 153 os.path.join(test_name, "index.html") + "\'>" + test_name + 154 "</a></pre></td>") 155 for field_name in ['functions', 'instantiations', 'lines', 'regions']: 156 field = test_totals[field_name] 157 totals[field_name]['covered'] += field['covered'] 158 totals[field_name]['count'] += field['count'] 159 write_root_html_column(f, field['covered'], field['count']) 160 f.write("</tr>"); 161 162 #Write the totals row. 163 f.write("<tr class='light-row-bold'><td><pre>Totals</a></pre></td>") 164 for field_name in ['functions', 'instantiations', 'lines', 'regions']: 165 field = totals[field_name] 166 write_root_html_column(f, field['covered'], field['count']) 167 f.write("</tr>"); 168 169 170def write_root_html_tail(f): 171 # Pulled from the generated html coverage report. 172 f.write("</table></div><h5>Generated by llvm-cov -- llvm version 7.0.2svn<" \ 173 "/h5></body></html>") 174 175 176def generate_root_html(tests): 177 # Copy the css file from one of the coverage reports. 178 source_file = os.path.join(os.path.join(WORKING_DIR, tests[0]['test_name']), "style.css") 179 dest_file = os.path.join(WORKING_DIR, "style.css") 180 shutil.copy2(source_file, dest_file) 181 182 # Write the root index.html file that sumarizes all the tests. 183 f = open(os.path.join(WORKING_DIR, "index.html"), "w") 184 write_root_html_head(f) 185 write_root_html_rows(f, tests) 186 write_root_html_tail(f) 187 188 189def get_profraw_for_test(test_name): 190 test_root = get_native_test_root_or_die() 191 test_cmd = os.path.join(os.path.join(test_root, test_name), test_name) 192 if not os.path.isfile(test_cmd): 193 logging.error('The test ' + test_name + ' does not exist, please compile first') 194 sys.exit(1) 195 196 profraw_file_name = test_name + ".profraw" 197 profraw_path = os.path.join(WORKING_DIR, os.path.join(test_name, profraw_file_name)) 198 llvm_env_var = "LLVM_PROFILE_FILE=\"" + profraw_path + "\"" 199 200 test_cmd = llvm_env_var + " " + test_cmd 201 logging.info('Generating profraw data for ' + test_name) 202 logging.debug('cmd: ' + test_cmd) 203 if subprocess.call(test_cmd, shell=True) != 0: 204 logging.error('Test ' + test_name + ' failed. Please fix the test before generating coverage.') 205 sys.exit(1) 206 207 if not os.path.isfile(profraw_path): 208 logging.error('Generating the profraw file failed. Did you remember to add the proper compiler flags to your build?') 209 sys.exit(1) 210 211 return profraw_file_name 212 213 214def merge_profraw_data(test_name): 215 cmd = [] 216 cmd.append(os.path.join(get_android_root_or_die(), LLVM_MERGE + " merge ")) 217 218 test_working_dir = os.path.join(WORKING_DIR, test_name); 219 cmd.append(os.path.join(test_working_dir, test_name + ".profraw")) 220 profdata_file = os.path.join(test_working_dir, test_name + ".profdata") 221 222 cmd.append('-o ' + profdata_file) 223 logging.info('Combining profraw files into profdata for ' + test_name) 224 logging.debug('cmd: ' + " ".join(cmd)) 225 if subprocess.call(" ".join(cmd), shell=True) != 0: 226 logging.error('Failed to merge profraw files for ' + test_name) 227 sys.exit(1) 228 229 230def generate_coverage_html(test): 231 COVERAGE_ROOT = '/proc/self/cwd' 232 233 test_name = test['test_name'] 234 file_list = test['covered_files'] 235 236 test_working_dir = os.path.join(WORKING_DIR, test_name) 237 test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata") 238 239 cmd = [ 240 os.path.join(get_android_root_or_die(), LLVM_COV), 241 "show", 242 "-format=html", 243 "-summary-only", 244 "-show-line-counts-or-regions", 245 "-show-instantiation-summary", 246 "-instr-profile=" + test_profdata_file, 247 "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" + 248 get_android_root_or_die() + "\"", 249 "-output-dir=" + test_working_dir 250 ] 251 252 # We have to have one object file not as an argument otherwise we can't specify source files. 253 test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name) 254 cmd.append(test_cmd) 255 256 # Filter out the specific files we want coverage for 257 for filename in file_list: 258 cmd.append(os.path.join(get_android_root_or_die(), filename)) 259 260 logging.info('Generating coverage report for ' + test['test_name']) 261 logging.debug('cmd: ' + " ".join(cmd)) 262 if subprocess.call(" ".join(cmd), shell=True) != 0: 263 logging.error('Failed to generate coverage for ' + test['test_name']) 264 sys.exit(1) 265 266 267def generate_coverage_json(test): 268 COVERAGE_ROOT = '/proc/self/cwd' 269 test_name = test['test_name'] 270 file_list = test['covered_files'] 271 272 test_working_dir = os.path.join(WORKING_DIR, test_name) 273 test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata") 274 275 cmd = [ 276 os.path.join(get_android_root_or_die(), LLVM_COV), 277 "export", 278 "-summary-only", 279 "-show-region-summary", 280 "-instr-profile=" + test_profdata_file, 281 "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" + get_android_root_or_die() + "\"", 282 ] 283 284 test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name) 285 cmd.append(test_cmd) 286 287 # Filter out the specific files we want coverage for 288 for filename in file_list: 289 cmd.append(os.path.join(get_android_root_or_die(), filename)) 290 291 logging.info('Generating coverage json for ' + test['test_name']) 292 logging.debug('cmd: ' + " ".join(cmd)) 293 294 json_str = subprocess.check_output(" ".join(cmd), shell=True) 295 return json.loads(json_str) 296 297 298def write_json_summary(test): 299 test_name = test['test_name'] 300 test_working_dir = os.path.join(WORKING_DIR, test_name) 301 test_json_summary_file = os.path.join(test_working_dir, test_name + '.json') 302 logging.debug('Writing json summary file: ' + test_json_summary_file) 303 json_file = open(test_json_summary_file, 'w') 304 json.dump(generate_coverage_json(test), json_file) 305 json_file.close() 306 307 308def list_tests(): 309 for test in COVERAGE_TESTS: 310 print "Test Name: " + test['test_name'] 311 print "Covered Files: " 312 for covered_file in test['covered_files']: 313 print " " + covered_file 314 print 315 316 317def main(): 318 parser = argparse.ArgumentParser(description='Generate code coverage for enabled tests.') 319 parser.add_argument( 320 '-l', '--list-tests', 321 action='store_true', 322 dest='list_tests', 323 help='List all the available tests to be run as well as covered files.') 324 parser.add_argument( 325 '-a', '--all', 326 action='store_true', 327 help='Runs all available tests and prints their outputs. If no tests ' \ 328 'are specified via the -t option all tests will be run.') 329 parser.add_argument( 330 '-t', '--test', 331 dest='tests', 332 action='append', 333 type=str, 334 metavar='TESTNAME', 335 default=[], 336 help='Specifies a test to be run. Multiple tests can be specified by ' \ 337 'using this option multiple times. ' \ 338 'Example: \"gen_coverage.py -t test1 -t test2\"') 339 parser.add_argument( 340 '-o', '--output', 341 type=str, 342 metavar='DIRECTORY', 343 default='/tmp/coverage', 344 help='Specifies the directory to store all files. The directory will be ' \ 345 'created if it does not exist. Default is \"/tmp/coverage\"') 346 parser.add_argument( 347 '-s', '--skip-html', 348 dest='skip_html', 349 action='store_true', 350 help='Skip opening up the results of the coverage report in a browser.') 351 parser.add_argument( 352 '-j', '--json-file', 353 dest='json_file', 354 action='store_true', 355 help='Write out summary results to json file in test directory.') 356 357 logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(levelname)s %(message)s') 358 logging.addLevelName(logging.DEBUG, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.DEBUG)) 359 logging.addLevelName(logging.INFO, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.INFO)) 360 logging.addLevelName(logging.WARNING, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.WARNING)) 361 logging.addLevelName(logging.ERROR, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.ERROR)) 362 363 args = parser.parse_args() 364 logging.debug("Args: " + str(args)) 365 366 # Set the working directory 367 global WORKING_DIR 368 WORKING_DIR = os.path.abspath(args.output) 369 logging.debug("Working Dir: " + WORKING_DIR) 370 371 # Print out the list of tests then exit 372 if args.list_tests: 373 list_tests() 374 sys.exit(0) 375 376 # Check to see if a test was specified and if so only generate coverage for 377 # that test. 378 if len(args.tests) == 0: 379 args.all = True 380 381 tests_to_run = [] 382 for test in COVERAGE_TESTS: 383 if args.all or test['test_name'] in args.tests: 384 tests_to_run.append(test) 385 if test['test_name'] in args.tests: 386 args.tests.remove(test['test_name']) 387 388 # Error if a test was specified but doesn't exist. 389 if len(args.tests) != 0: 390 for test_name in args.tests: 391 logging.error('\"' + test_name + '\" was not found in the list of available tests.') 392 sys.exit(1) 393 394 # Generate the info for the tests 395 for test in tests_to_run: 396 logging.info('Getting coverage for ' + test['test_name']) 397 get_profraw_for_test(test['test_name']) 398 merge_profraw_data(test['test_name']) 399 if args.json_file: 400 write_json_summary(test) 401 generate_coverage_html(test) 402 403 # Generate the root index.html page that sumarizes all of the coverage reports. 404 generate_root_html(tests_to_run) 405 406 # Open the results in a browser. 407 if not args.skip_html: 408 webbrowser.open('file://' + os.path.join(WORKING_DIR, 'index.html')) 409 410 411if __name__ == '__main__': 412 main() 413