1#!/usr/bin/env python 2# Copyright 2017 The PDFium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Generates a coverage report for given binaries using llvm-gcov & lcov. 7 8Requires llvm-cov 3.5 or later. 9Requires lcov 1.11 or later. 10Requires that 'use_coverage = true' is set in args.gn. 11""" 12 13import argparse 14from collections import namedtuple 15import os 16import pprint 17import re 18import subprocess 19import sys 20 21 22# Add src dir to path to avoid having to set PYTHONPATH. 23sys.path.append( 24 os.path.abspath( 25 os.path.join( 26 os.path.dirname(__file__), 27 os.path.pardir, 28 os.path.pardir, 29 os.path.pardir))) 30 31from testing.tools.common import GetBooleanGnArg 32 33 34# 'binary' is the file that is to be run for the test. 35# 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus 36# requires special handling. 37TestSpec = namedtuple('TestSpec', 'binary, use_test_runner') 38 39# All of the coverage tests that the script knows how to run. 40COVERAGE_TESTS = { 41 'pdfium_unittests': TestSpec('pdfium_unittests', False), 42 'pdfium_embeddertests': TestSpec('pdfium_embeddertests', False), 43 'corpus_tests': TestSpec('run_corpus_tests.py', True), 44 'javascript_tests': TestSpec('run_javascript_tests.py', True), 45 'pixel_tests': TestSpec('run_pixel_tests.py', True), 46} 47 48# Coverage tests that are known to take a long time to run, so are not in the 49# default set. The user must either explicitly invoke these tests or pass in 50# --slow. 51SLOW_TESTS = ['corpus_tests', 'javascript_tests', 'pixel_tests'] 52 53class CoverageExecutor(object): 54 55 def __init__(self, parser, args): 56 """Initialize executor based on the current script environment 57 58 Args: 59 parser: argparse.ArgumentParser for handling improper inputs. 60 args: Dictionary of arguments passed into the calling script. 61 """ 62 self.dry_run = args['dry_run'] 63 self.verbose = args['verbose'] 64 65 llvm_cov = self.determine_proper_llvm_cov() 66 if not llvm_cov: 67 print 'Unable to find appropriate llvm-cov to use' 68 sys.exit(1) 69 self.lcov_env = os.environ 70 self.lcov_env['LLVM_COV_BIN'] = llvm_cov 71 72 self.lcov = self.determine_proper_lcov() 73 if not self.lcov: 74 print 'Unable to find appropriate lcov to use' 75 sys.exit(1) 76 77 self.coverage_files = set() 78 self.source_directory = args['source_directory'] 79 if not os.path.isdir(self.source_directory): 80 parser.error("'%s' needs to be a directory" % self.source_directory) 81 82 self.build_directory = args['build_directory'] 83 if not os.path.isdir(self.build_directory): 84 parser.error("'%s' needs to be a directory" % self.build_directory) 85 86 self.coverage_tests = self.calculate_coverage_tests(args) 87 if not self.coverage_tests: 88 parser.error( 89 'No valid tests in set to be run. This is likely due to bad command ' 90 'line arguments') 91 92 if not GetBooleanGnArg('use_coverage', self.build_directory, self.verbose): 93 parser.error( 94 'use_coverage does not appear to be set to true for build, but is ' 95 'needed') 96 97 self.use_goma = GetBooleanGnArg('use_goma', self.build_directory, 98 self.verbose) 99 100 self.output_directory = args['output_directory'] 101 if not os.path.exists(self.output_directory): 102 if not self.dry_run: 103 os.makedirs(self.output_directory) 104 elif not os.path.isdir(self.output_directory): 105 parser.error('%s exists, but is not a directory' % self.output_directory) 106 self.coverage_totals_path = os.path.join(self.output_directory, 107 'pdfium_totals.info') 108 109 def check_output(self, args, dry_run=False, env=None): 110 """Dry run aware wrapper of subprocess.check_output()""" 111 if dry_run: 112 print "Would have run '%s'" % ' '.join(args) 113 return '' 114 115 output = subprocess.check_output(args, env=env) 116 117 if self.verbose: 118 print "check_output(%s) returned '%s'" % (args, output) 119 return output 120 121 def call(self, args, dry_run=False, env=None): 122 """Dry run aware wrapper of subprocess.call()""" 123 if dry_run: 124 print "Would have run '%s'" % ' '.join(args) 125 return 0 126 127 output = subprocess.call(args, env=env) 128 129 if self.verbose: 130 print 'call(%s) returned %s' % (args, output) 131 return output 132 133 def call_lcov(self, args, dry_run=False, needs_directory=True): 134 """Wrapper to call lcov that adds appropriate arguments as needed.""" 135 lcov_args = [ 136 self.lcov, '--config-file', 137 os.path.join(self.source_directory, 'testing', 'tools', 'coverage', 138 'lcovrc'), 139 '--gcov-tool', 140 os.path.join(self.source_directory, 'testing', 'tools', 'coverage', 141 'llvm-gcov') 142 ] 143 if needs_directory: 144 lcov_args.extend(['--directory', self.source_directory]) 145 if not self.verbose: 146 lcov_args.append('--quiet') 147 lcov_args.extend(args) 148 return self.call(lcov_args, dry_run=dry_run, env=self.lcov_env) 149 150 def calculate_coverage_tests(self, args): 151 """Determine which tests should be run.""" 152 testing_tools_directory = os.path.join(self.source_directory, 'testing', 153 'tools') 154 coverage_tests = {} 155 for name in COVERAGE_TESTS.keys(): 156 test_spec = COVERAGE_TESTS[name] 157 if test_spec.use_test_runner: 158 binary_path = os.path.join(testing_tools_directory, test_spec.binary) 159 else: 160 binary_path = os.path.join(self.build_directory, test_spec.binary) 161 coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner) 162 163 if args['tests']: 164 return {name: spec 165 for name, spec in coverage_tests.iteritems() if name in args['tests']} 166 elif not args['slow']: 167 return {name: spec 168 for name, spec in coverage_tests.iteritems() if name not in SLOW_TESTS} 169 else: 170 return coverage_tests 171 172 def find_acceptable_binary(self, binary_name, version_regex, 173 min_major_version, min_minor_version): 174 """Find the newest version of binary that meets the min version.""" 175 min_version = (min_major_version, min_minor_version) 176 parsed_versions = {} 177 # When calling Bash builtins like this the command and arguments must be 178 # passed in as a single string instead of as separate list members. 179 potential_binaries = self.check_output( 180 ['bash', '-c', 'compgen -abck %s' % binary_name]).splitlines() 181 for binary in potential_binaries: 182 if self.verbose: 183 print 'Testing llvm-cov binary, %s' % binary 184 # Assuming that scripts that don't respond to --version correctly are not 185 # valid binaries and just happened to get globbed in. This is true for 186 # lcov and llvm-cov 187 try: 188 version_output = self.check_output([binary, '--version']).splitlines() 189 except subprocess.CalledProcessError: 190 if self.verbose: 191 print '--version returned failure status 1, so ignoring' 192 continue 193 194 for line in version_output: 195 matcher = re.match(version_regex, line) 196 if matcher: 197 parsed_version = (int(matcher.group(1)), int(matcher.group(2))) 198 if parsed_version >= min_version: 199 parsed_versions[parsed_version] = binary 200 break 201 202 if not parsed_versions: 203 return None 204 return parsed_versions[max(parsed_versions)] 205 206 def determine_proper_llvm_cov(self): 207 """Find a version of llvm_cov that will work with the script.""" 208 version_regex = re.compile('.*LLVM version ([\d]+)\.([\d]+).*') 209 return self.find_acceptable_binary('llvm-cov', version_regex, 3, 5) 210 211 def determine_proper_lcov(self): 212 """Find a version of lcov that will work with the script.""" 213 version_regex = re.compile('.*LCOV version ([\d]+)\.([\d]+).*') 214 return self.find_acceptable_binary('lcov', version_regex, 1, 11) 215 216 def build_binaries(self): 217 """Build all the binaries that are going to be needed for coverage 218 generation.""" 219 call_args = ['ninja'] 220 if self.use_goma: 221 call_args.extend(['-j', '250']) 222 call_args.extend(['-C', self.build_directory]) 223 return self.call(call_args, dry_run=self.dry_run) == 0 224 225 def generate_coverage(self, name, spec): 226 """Generate the coverage data for a test 227 228 Args: 229 name: Name associated with the test to be run. This is used as a label 230 in the coverage data, so should be unique across all of the tests 231 being run. 232 spec: Tuple containing the path to the binary to run, and if this test 233 uses test_runner.py. 234 """ 235 if self.verbose: 236 print "Generating coverage for test '%s', using data '%s'" % (name, spec) 237 if not os.path.exists(spec.binary): 238 print('Unable to generate coverage for %s, since it appears to not exist' 239 ' @ %s') % (name, spec.binary) 240 return False 241 242 if self.call_lcov(['--zerocounters'], dry_run=self.dry_run): 243 print 'Unable to clear counters for %s' % name 244 return False 245 246 binary_args = [spec.binary] 247 if spec.use_test_runner: 248 # Test runner performs multi-threading in the wrapper script, not the test 249 # binary, so need -j 1, otherwise multiple processes will be writing to 250 # the code coverage files, invalidating results. 251 # TODO(pdfium:811): Rewrite how test runner tests work, so that they can 252 # be run in multi-threaded mode. 253 binary_args.extend(['-j', '1', '--build-dir', self.build_directory]) 254 if self.call(binary_args, dry_run=self.dry_run) and self.verbose: 255 print('Running %s appears to have failed, which might affect ' 256 'results') % spec.binary 257 258 output_raw_path = os.path.join(self.output_directory, '%s_raw.info' % name) 259 if self.call_lcov( 260 ['--capture', '--test-name', name, '--output-file', output_raw_path], 261 dry_run=self.dry_run): 262 print 'Unable to capture coverage data for %s' % name 263 return False 264 265 output_filtered_path = os.path.join(self.output_directory, 266 '%s_filtered.info' % name) 267 output_filters = [ 268 '/usr/include/*', '*third_party*', '*testing*', '*_unittest.cpp', 269 '*_embeddertest.cpp' 270 ] 271 if self.call_lcov( 272 ['--remove', output_raw_path] + output_filters + 273 ['--output-file', output_filtered_path], 274 dry_run=self.dry_run, 275 needs_directory=False): 276 print 'Unable to filter coverage data for %s' % name 277 return False 278 279 self.coverage_files.add(output_filtered_path) 280 return True 281 282 def merge_coverage(self): 283 """Merge all of the coverage data sets into one for report generation.""" 284 merge_args = [] 285 for coverage_file in self.coverage_files: 286 merge_args.extend(['--add-tracefile', coverage_file]) 287 288 merge_args.extend(['--output-file', self.coverage_totals_path]) 289 return self.call_lcov( 290 merge_args, dry_run=self.dry_run, needs_directory=False) == 0 291 292 def generate_report(self): 293 """Produce HTML coverage report based on combined coverage data set.""" 294 config_file = os.path.join( 295 self.source_directory, 'testing', 'tools', 'coverage', 'lcovrc') 296 297 lcov_args = ['genhtml', 298 '--config-file', config_file, 299 '--legend', 300 '--demangle-cpp', 301 '--show-details', 302 '--prefix', self.source_directory, 303 '--ignore-errors', 304 'source', self.coverage_totals_path, 305 '--output-directory', self.output_directory] 306 return self.call(lcov_args, dry_run=self.dry_run) == 0 307 308 def run(self): 309 """Setup environment, execute the tests and generate coverage report""" 310 if not self.build_binaries(): 311 print 'Failed to successfully build binaries' 312 return False 313 314 for name in self.coverage_tests.keys(): 315 if not self.generate_coverage(name, self.coverage_tests[name]): 316 print 'Failed to successfully generate coverage data' 317 return False 318 319 if not self.merge_coverage(): 320 print 'Failed to successfully merge generated coverage data' 321 return False 322 323 if not self.generate_report(): 324 print 'Failed to successfully generated coverage report' 325 return False 326 327 return True 328 329 330def main(): 331 parser = argparse.ArgumentParser() 332 parser.formatter_class = argparse.RawDescriptionHelpFormatter 333 parser.description = ('Generates a coverage report for given binaries using ' 334 'llvm-cov & lcov.\n\n' 335 'Requires llvm-cov 3.5 or later.\n' 336 'Requires lcov 1.11 or later.\n\n' 337 'By default runs pdfium_unittests and ' 338 'pdfium_embeddertests. If --slow is passed in then all ' 339 'tests will be run. If any of the tests are specified ' 340 'on the command line, then only those will be run.') 341 parser.add_argument( 342 '-s', 343 '--source_directory', 344 help='Location of PDFium source directory, defaults to CWD', 345 default=os.getcwd()) 346 build_default = os.path.join('out', 'Coverage') 347 parser.add_argument( 348 '-b', 349 '--build_directory', 350 help= 351 'Location of PDFium build directory with coverage enabled, defaults to ' 352 '%s under CWD' % build_default, 353 default=os.path.join(os.getcwd(), build_default)) 354 output_default = 'coverage_report' 355 parser.add_argument( 356 '-o', 357 '--output_directory', 358 help='Location to write out coverage report to, defaults to %s under CWD ' 359 % output_default, 360 default=os.path.join(os.getcwd(), output_default)) 361 parser.add_argument( 362 '-n', 363 '--dry-run', 364 help='Output commands instead of executing them', 365 action='store_true') 366 parser.add_argument( 367 '-v', 368 '--verbose', 369 help='Output additional diagnostic information', 370 action='store_true') 371 parser.add_argument( 372 '--slow', 373 help='Run all tests, even those known to take a long time. Ignored if ' 374 'specific tests are passed in.', 375 action='store_true') 376 parser.add_argument( 377 'tests', 378 help='Tests to be run, defaults to all. Valid entries are %s' % 379 COVERAGE_TESTS.keys(), 380 nargs='*') 381 382 args = vars(parser.parse_args()) 383 if args['verbose']: 384 pprint.pprint(args) 385 386 executor = CoverageExecutor(parser, args) 387 if executor.run(): 388 return 0 389 return 1 390 391 392if __name__ == '__main__': 393 sys.exit(main()) 394