1#!/usr/bin/env python3 2# 3# Copyright 2013 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""Runs Android's lint tool.""" 7 8import argparse 9import logging 10import os 11import shutil 12import sys 13import time 14from xml.dom import minidom 15from xml.etree import ElementTree 16 17from util import build_utils 18from util import manifest_utils 19from util import server_utils 20import action_helpers # build_utils adds //build to sys.path. 21 22_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/main/build/android/docs/lint.md' # pylint: disable=line-too-long 23 24# These checks are not useful for chromium. 25_DISABLED_ALWAYS = [ 26 "AppCompatResource", # Lint does not correctly detect our appcompat lib. 27 "Assert", # R8 --force-enable-assertions is used to enable java asserts. 28 "InflateParams", # Null is ok when inflating views for dialogs. 29 "InlinedApi", # Constants are copied so they are always available. 30 "LintBaseline", # Don't warn about using baseline.xml files. 31 "MissingInflatedId", # False positives https://crbug.com/1394222 32 "MissingApplicationIcon", # False positive for non-production targets. 33 "NetworkSecurityConfig", # Breaks on library certificates b/269783280. 34 "ObsoleteLintCustomCheck", # We have no control over custom lint checks. 35 "SwitchIntDef", # Many C++ enums are not used at all in java. 36 "Typos", # Strings are committed in English first and later translated. 37 "VisibleForTests", # Does not recognize "ForTesting" methods. 38 "UniqueConstants", # Chromium enums allow aliases. 39 "UnusedAttribute", # Chromium apks have various minSdkVersion values. 40] 41 42# These checks are not useful for test targets and adds an unnecessary burden 43# to suppress them. 44_DISABLED_FOR_TESTS = [ 45 # We should not require test strings.xml files to explicitly add 46 # translatable=false since they are not translated and not used in 47 # production. 48 "MissingTranslation", 49 # Test strings.xml files often have simple names and are not translatable, 50 # so it may conflict with a production string and cause this error. 51 "Untranslatable", 52 # Test targets often use the same strings target and resources target as the 53 # production targets but may not use all of them. 54 "UnusedResources", 55 # TODO(wnwen): Turn this back on since to crash it would require running on 56 # a device with all the various minSdkVersions. 57 # Real NewApi violations crash the app, so the only ones that lint catches 58 # but tests still succeed are false positives. 59 "NewApi", 60 # Tests should be allowed to access these methods/classes. 61 "VisibleForTests", 62] 63 64_RES_ZIP_DIR = 'RESZIPS' 65_SRCJAR_DIR = 'SRCJARS' 66_AAR_DIR = 'AARS' 67 68 69def _SrcRelative(path): 70 """Returns relative path to top-level src dir.""" 71 return os.path.relpath(path, build_utils.DIR_SOURCE_ROOT) 72 73 74def _GenerateProjectFile(android_manifest, 75 android_sdk_root, 76 cache_dir, 77 sources=None, 78 classpath=None, 79 srcjar_sources=None, 80 resource_sources=None, 81 custom_lint_jars=None, 82 custom_annotation_zips=None, 83 android_sdk_version=None, 84 baseline_path=None): 85 project = ElementTree.Element('project') 86 root = ElementTree.SubElement(project, 'root') 87 # Run lint from output directory: crbug.com/1115594 88 root.set('dir', os.getcwd()) 89 sdk = ElementTree.SubElement(project, 'sdk') 90 # Lint requires that the sdk path be an absolute path. 91 sdk.set('dir', os.path.abspath(android_sdk_root)) 92 if baseline_path is not None: 93 baseline = ElementTree.SubElement(project, 'baseline') 94 baseline.set('file', baseline_path) 95 cache = ElementTree.SubElement(project, 'cache') 96 cache.set('dir', cache_dir) 97 main_module = ElementTree.SubElement(project, 'module') 98 main_module.set('name', 'main') 99 main_module.set('android', 'true') 100 main_module.set('library', 'false') 101 if android_sdk_version: 102 main_module.set('compile_sdk_version', android_sdk_version) 103 manifest = ElementTree.SubElement(main_module, 'manifest') 104 manifest.set('file', android_manifest) 105 if srcjar_sources: 106 for srcjar_file in srcjar_sources: 107 src = ElementTree.SubElement(main_module, 'src') 108 src.set('file', srcjar_file) 109 if sources: 110 for source in sources: 111 src = ElementTree.SubElement(main_module, 'src') 112 src.set('file', source) 113 if classpath: 114 for file_path in classpath: 115 classpath_element = ElementTree.SubElement(main_module, 'classpath') 116 classpath_element.set('file', file_path) 117 if resource_sources: 118 for resource_file in resource_sources: 119 resource = ElementTree.SubElement(main_module, 'resource') 120 resource.set('file', resource_file) 121 if custom_lint_jars: 122 for lint_jar in custom_lint_jars: 123 lint = ElementTree.SubElement(main_module, 'lint-checks') 124 lint.set('file', lint_jar) 125 if custom_annotation_zips: 126 for annotation_zip in custom_annotation_zips: 127 annotation = ElementTree.SubElement(main_module, 'annotations') 128 annotation.set('file', annotation_zip) 129 return project 130 131 132def _RetrieveBackportedMethods(backported_methods_path): 133 with open(backported_methods_path) as f: 134 methods = f.read().splitlines() 135 # Methods look like: 136 # java/util/Set#of(Ljava/lang/Object;)Ljava/util/Set; 137 # But error message looks like: 138 # Call requires API level R (current min is 21): java.util.Set#of [NewApi] 139 methods = (m.replace('/', '\\.') for m in methods) 140 methods = (m[:m.index('(')] for m in methods) 141 return sorted(set(methods)) 142 143 144def _GenerateConfigXmlTree(orig_config_path, backported_methods): 145 if orig_config_path: 146 root_node = ElementTree.parse(orig_config_path).getroot() 147 else: 148 root_node = ElementTree.fromstring('<lint/>') 149 150 issue_node = ElementTree.SubElement(root_node, 'issue') 151 issue_node.attrib['id'] = 'NewApi' 152 ignore_node = ElementTree.SubElement(issue_node, 'ignore') 153 ignore_node.attrib['regexp'] = '|'.join(backported_methods) 154 return root_node 155 156 157def _GenerateAndroidManifest(original_manifest_path, extra_manifest_paths, 158 min_sdk_version, android_sdk_version): 159 # Set minSdkVersion in the manifest to the correct value. 160 doc, manifest, app_node = manifest_utils.ParseManifest(original_manifest_path) 161 162 # TODO(crbug.com/1126301): Should this be done using manifest merging? 163 # Add anything in the application node of the extra manifests to the main 164 # manifest to prevent unused resource errors. 165 for path in extra_manifest_paths: 166 _, _, extra_app_node = manifest_utils.ParseManifest(path) 167 for node in extra_app_node: 168 app_node.append(node) 169 170 uses_sdk = manifest.find('./uses-sdk') 171 if uses_sdk is None: 172 uses_sdk = ElementTree.Element('uses-sdk') 173 manifest.insert(0, uses_sdk) 174 uses_sdk.set('{%s}minSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 175 min_sdk_version) 176 uses_sdk.set('{%s}targetSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 177 android_sdk_version) 178 return doc 179 180 181def _WriteXmlFile(root, path): 182 logging.info('Writing xml file %s', path) 183 build_utils.MakeDirectory(os.path.dirname(path)) 184 with action_helpers.atomic_output(path) as f: 185 # Although we can write it just with ElementTree.tostring, using minidom 186 # makes it a lot easier to read as a human (also on code search). 187 f.write( 188 minidom.parseString(ElementTree.tostring( 189 root, encoding='utf-8')).toprettyxml(indent=' ').encode('utf-8')) 190 191 192def _RunLint(create_cache, 193 custom_lint_jar_path, 194 lint_jar_path, 195 backported_methods_path, 196 config_path, 197 manifest_path, 198 extra_manifest_paths, 199 sources, 200 classpath, 201 cache_dir, 202 android_sdk_version, 203 aars, 204 srcjars, 205 min_sdk_version, 206 resource_sources, 207 resource_zips, 208 android_sdk_root, 209 lint_gen_dir, 210 baseline, 211 testonly_target=False, 212 warnings_as_errors=False): 213 logging.info('Lint starting') 214 215 if create_cache: 216 # Occasionally lint may crash due to re-using intermediate files from older 217 # lint runs. See https://crbug.com/1258178 for context. 218 logging.info('Clearing cache dir %s before creating cache.', cache_dir) 219 shutil.rmtree(cache_dir, ignore_errors=True) 220 os.makedirs(cache_dir) 221 222 if baseline and not os.path.exists(baseline): 223 # Generating new baselines is only done locally, and requires more memory to 224 # avoid OOMs. 225 creating_baseline = True 226 lint_xmx = '4G' 227 else: 228 creating_baseline = False 229 lint_xmx = '2G' 230 231 # All paths in lint are based off of relative paths from root with root as the 232 # prefix. Path variable substitution is based off of prefix matching so custom 233 # path variables need to match exactly in order to show up in baseline files. 234 # e.g. lint_path=path/to/output/dir/../../file/in/src 235 root_path = os.getcwd() # This is usually the output directory. 236 pathvar_src = os.path.join( 237 root_path, os.path.relpath(build_utils.DIR_SOURCE_ROOT, start=root_path)) 238 239 cmd = build_utils.JavaCmd(xmx=lint_xmx) + [ 240 '-cp', 241 '{}:{}'.format(lint_jar_path, custom_lint_jar_path), 242 'org.chromium.build.CustomLint', 243 '--sdk-home', 244 android_sdk_root, 245 '--jdk-home', 246 build_utils.JAVA_HOME, 247 '--path-variables', 248 f'SRC={pathvar_src}', 249 '--quiet', # Silences lint's "." progress updates. 250 '--stacktrace', # Prints full stacktraces for internal lint errors. 251 '--disable', 252 ','.join(_DISABLED_ALWAYS), 253 ] 254 255 if testonly_target: 256 cmd.extend(['--disable', ','.join(_DISABLED_FOR_TESTS)]) 257 258 if not manifest_path: 259 manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build', 260 'android', 'AndroidManifest.xml') 261 262 logging.info('Generating config.xml') 263 backported_methods = _RetrieveBackportedMethods(backported_methods_path) 264 config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods) 265 generated_config_path = os.path.join(lint_gen_dir, 'config.xml') 266 _WriteXmlFile(config_xml_node, generated_config_path) 267 cmd.extend(['--config', generated_config_path]) 268 269 logging.info('Generating Android manifest file') 270 android_manifest_tree = _GenerateAndroidManifest(manifest_path, 271 extra_manifest_paths, 272 min_sdk_version, 273 android_sdk_version) 274 # Just use a hardcoded name, since we may have different target names (and 275 # thus different manifest_paths) using the same lint baseline. Eg. 276 # trichrome_chrome_bundle and trichrome_chrome_32_64_bundle. 277 lint_android_manifest_path = os.path.join(lint_gen_dir, 'AndroidManifest.xml') 278 _WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path) 279 280 resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR) 281 # These are zip files with generated resources (e. g. strings from GRD). 282 logging.info('Extracting resource zips') 283 for resource_zip in resource_zips: 284 # Use a consistent root and name rather than a temporary file so that 285 # suppressions can be local to the lint target and the resource target. 286 resource_dir = os.path.join(resource_root_dir, resource_zip) 287 shutil.rmtree(resource_dir, True) 288 os.makedirs(resource_dir) 289 resource_sources.extend( 290 build_utils.ExtractAll(resource_zip, path=resource_dir)) 291 292 logging.info('Extracting aars') 293 aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR) 294 custom_lint_jars = [] 295 custom_annotation_zips = [] 296 if aars: 297 for aar in aars: 298 # Use relative source for aar files since they are not generated. 299 aar_dir = os.path.join(aar_root_dir, 300 os.path.splitext(_SrcRelative(aar))[0]) 301 shutil.rmtree(aar_dir, True) 302 os.makedirs(aar_dir) 303 aar_files = build_utils.ExtractAll(aar, path=aar_dir) 304 for f in aar_files: 305 if f.endswith('lint.jar'): 306 custom_lint_jars.append(f) 307 elif f.endswith('annotations.zip'): 308 custom_annotation_zips.append(f) 309 310 logging.info('Extracting srcjars') 311 srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR) 312 srcjar_sources = [] 313 if srcjars: 314 for srcjar in srcjars: 315 # Use path without extensions since otherwise the file name includes 316 # .srcjar and lint treats it as a srcjar. 317 srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0]) 318 shutil.rmtree(srcjar_dir, True) 319 os.makedirs(srcjar_dir) 320 # Sadly lint's srcjar support is broken since it only considers the first 321 # srcjar. Until we roll a lint version with that fixed, we need to extract 322 # it ourselves. 323 srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir)) 324 325 logging.info('Generating project file') 326 project_file_root = _GenerateProjectFile(lint_android_manifest_path, 327 android_sdk_root, cache_dir, sources, 328 classpath, srcjar_sources, 329 resource_sources, custom_lint_jars, 330 custom_annotation_zips, 331 android_sdk_version, baseline) 332 333 project_xml_path = os.path.join(lint_gen_dir, 'project.xml') 334 _WriteXmlFile(project_file_root, project_xml_path) 335 cmd += ['--project', project_xml_path] 336 337 # This filter is necessary for JDK11. 338 stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings 339 stdout_filter = lambda x: build_utils.FilterLines(x, 'No issues found') 340 341 start = time.time() 342 logging.debug('Lint command %s', ' '.join(cmd)) 343 failed = False 344 345 if creating_baseline and not warnings_as_errors: 346 # Allow error code 6 when creating a baseline: ERRNO_CREATED_BASELINE 347 fail_func = lambda returncode, _: returncode not in (0, 6) 348 else: 349 fail_func = lambda returncode, _: returncode != 0 350 351 try: 352 build_utils.CheckOutput(cmd, 353 print_stdout=True, 354 stdout_filter=stdout_filter, 355 stderr_filter=stderr_filter, 356 fail_on_output=warnings_as_errors, 357 fail_func=fail_func) 358 except build_utils.CalledProcessError as e: 359 failed = True 360 # Do not output the python stacktrace because it is lengthy and is not 361 # relevant to the actual lint error. 362 sys.stderr.write(e.output) 363 finally: 364 # When not treating warnings as errors, display the extra footer. 365 is_debug = os.environ.get('LINT_DEBUG', '0') != '0' 366 367 end = time.time() - start 368 logging.info('Lint command took %ss', end) 369 if not is_debug: 370 shutil.rmtree(aar_root_dir, ignore_errors=True) 371 shutil.rmtree(resource_root_dir, ignore_errors=True) 372 shutil.rmtree(srcjar_root_dir, ignore_errors=True) 373 os.unlink(project_xml_path) 374 375 if failed: 376 print('- For more help with lint in Chrome:', _LINT_MD_URL) 377 if is_debug: 378 print('- DEBUG MODE: Here is the project.xml: {}'.format( 379 _SrcRelative(project_xml_path))) 380 else: 381 print('- Run with LINT_DEBUG=1 to enable lint configuration debugging') 382 sys.exit(1) 383 384 logging.info('Lint completed') 385 386 387def _ParseArgs(argv): 388 parser = argparse.ArgumentParser() 389 action_helpers.add_depfile_arg(parser) 390 parser.add_argument('--target-name', help='Fully qualified GN target name.') 391 parser.add_argument('--skip-build-server', 392 action='store_true', 393 help='Avoid using the build server.') 394 parser.add_argument('--use-build-server', 395 action='store_true', 396 help='Always use the build server.') 397 parser.add_argument('--lint-jar-path', 398 required=True, 399 help='Path to the lint jar.') 400 parser.add_argument('--custom-lint-jar-path', 401 required=True, 402 help='Path to our custom lint jar.') 403 parser.add_argument('--backported-methods', 404 help='Path to backported methods file created by R8.') 405 parser.add_argument('--cache-dir', 406 required=True, 407 help='Path to the directory in which the android cache ' 408 'directory tree should be stored.') 409 parser.add_argument('--config-path', help='Path to lint suppressions file.') 410 parser.add_argument('--lint-gen-dir', 411 required=True, 412 help='Path to store generated xml files.') 413 parser.add_argument('--stamp', help='Path to stamp upon success.') 414 parser.add_argument('--android-sdk-version', 415 help='Version (API level) of the Android SDK used for ' 416 'building.') 417 parser.add_argument('--min-sdk-version', 418 required=True, 419 help='Minimal SDK version to lint against.') 420 parser.add_argument('--android-sdk-root', 421 required=True, 422 help='Lint needs an explicit path to the android sdk.') 423 parser.add_argument('--testonly', 424 action='store_true', 425 help='If set, some checks like UnusedResources will be ' 426 'disabled since they are not helpful for test ' 427 'targets.') 428 parser.add_argument('--create-cache', 429 action='store_true', 430 help='Whether this invocation is just warming the cache.') 431 parser.add_argument('--warnings-as-errors', 432 action='store_true', 433 help='Treat all warnings as errors.') 434 parser.add_argument('--sources', 435 help='A list of files containing java and kotlin source ' 436 'files.') 437 parser.add_argument('--aars', help='GN list of included aars.') 438 parser.add_argument('--srcjars', help='GN list of included srcjars.') 439 parser.add_argument('--manifest-path', 440 help='Path to original AndroidManifest.xml') 441 parser.add_argument('--extra-manifest-paths', 442 action='append', 443 help='GYP-list of manifest paths to merge into the ' 444 'original AndroidManifest.xml') 445 parser.add_argument('--resource-sources', 446 default=[], 447 action='append', 448 help='GYP-list of resource sources files, similar to ' 449 'java sources files, but for resource files.') 450 parser.add_argument('--resource-zips', 451 default=[], 452 action='append', 453 help='GYP-list of resource zips, zip files of generated ' 454 'resource files.') 455 parser.add_argument('--classpath', 456 help='List of jars to add to the classpath.') 457 parser.add_argument('--baseline', 458 help='Baseline file to ignore existing errors and fail ' 459 'on new errors.') 460 461 args = parser.parse_args(build_utils.ExpandFileArgs(argv)) 462 args.sources = action_helpers.parse_gn_list(args.sources) 463 args.aars = action_helpers.parse_gn_list(args.aars) 464 args.srcjars = action_helpers.parse_gn_list(args.srcjars) 465 args.resource_sources = action_helpers.parse_gn_list(args.resource_sources) 466 args.extra_manifest_paths = action_helpers.parse_gn_list( 467 args.extra_manifest_paths) 468 args.resource_zips = action_helpers.parse_gn_list(args.resource_zips) 469 args.classpath = action_helpers.parse_gn_list(args.classpath) 470 471 if args.baseline: 472 assert os.path.basename(args.baseline) == 'lint-baseline.xml', ( 473 'The baseline file needs to be named "lint-baseline.xml" in order for ' 474 'the autoroller to find and update it whenever lint is rolled to a new ' 475 'version.') 476 477 return args 478 479 480def main(): 481 build_utils.InitLogging('LINT_DEBUG') 482 args = _ParseArgs(sys.argv[1:]) 483 484 # TODO(wnwen): Consider removing lint cache now that there are only two lint 485 # invocations. 486 # Avoid parallelizing cache creation since lint runs without the cache defeat 487 # the purpose of creating the cache in the first place. 488 if (not args.create_cache and not args.skip_build_server 489 and server_utils.MaybeRunCommand(name=args.target_name, 490 argv=sys.argv, 491 stamp_file=args.stamp, 492 force=args.use_build_server)): 493 return 494 495 sources = [] 496 for sources_file in args.sources: 497 sources.extend(build_utils.ReadSourcesList(sources_file)) 498 resource_sources = [] 499 for resource_sources_file in args.resource_sources: 500 resource_sources.extend(build_utils.ReadSourcesList(resource_sources_file)) 501 502 possible_depfile_deps = (args.srcjars + args.resource_zips + sources + 503 resource_sources + [ 504 args.baseline, 505 args.manifest_path, 506 ]) 507 depfile_deps = [p for p in possible_depfile_deps if p] 508 509 _RunLint(args.create_cache, 510 args.custom_lint_jar_path, 511 args.lint_jar_path, 512 args.backported_methods, 513 args.config_path, 514 args.manifest_path, 515 args.extra_manifest_paths, 516 sources, 517 args.classpath, 518 args.cache_dir, 519 args.android_sdk_version, 520 args.aars, 521 args.srcjars, 522 args.min_sdk_version, 523 resource_sources, 524 args.resource_zips, 525 args.android_sdk_root, 526 args.lint_gen_dir, 527 args.baseline, 528 testonly_target=args.testonly, 529 warnings_as_errors=args.warnings_as_errors) 530 logging.info('Creating stamp file') 531 build_utils.Touch(args.stamp) 532 533 if args.depfile: 534 action_helpers.write_depfile(args.depfile, args.stamp, depfile_deps) 535 536 537if __name__ == '__main__': 538 sys.exit(main()) 539