1#!/usr/bin/env python3 2# 3# Copyright 2013 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""Runs Android's lint tool.""" 7 8import argparse 9import logging 10import os 11import shutil 12import sys 13import time 14from xml.dom import minidom 15from xml.etree import ElementTree 16 17from util import build_utils 18from util import manifest_utils 19from util import server_utils 20import action_helpers # build_utils adds //build to sys.path. 21 22_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/main/build/android/docs/lint.md' # pylint: disable=line-too-long 23 24# These checks are not useful for chromium. 25_DISABLED_ALWAYS = [ 26 "AppCompatResource", # Lint does not correctly detect our appcompat lib. 27 "Assert", # R8 --force-enable-assertions is used to enable java asserts. 28 "InflateParams", # Null is ok when inflating views for dialogs. 29 "InlinedApi", # Constants are copied so they are always available. 30 "LintBaseline", # Don't warn about using baseline.xml files. 31 "LintBaselineFixed", # We dont care if baseline.xml has unused entries. 32 "MissingInflatedId", # False positives https://crbug.com/1394222 33 "MissingApplicationIcon", # False positive for non-production targets. 34 "NetworkSecurityConfig", # Breaks on library certificates b/269783280. 35 "ObsoleteLintCustomCheck", # We have no control over custom lint checks. 36 "OldTargetApi", # We sometimes need targetSdkVersion to not be latest. 37 "StringFormatCount", # Has false-positives. 38 "SwitchIntDef", # Many C++ enums are not used at all in java. 39 "Typos", # Strings are committed in English first and later translated. 40 "VisibleForTests", # Does not recognize "ForTesting" methods. 41 "UniqueConstants", # Chromium enums allow aliases. 42 "UnusedAttribute", # Chromium apks have various minSdkVersion values. 43] 44 45_RES_ZIP_DIR = 'RESZIPS' 46_SRCJAR_DIR = 'SRCJARS' 47_AAR_DIR = 'AARS' 48 49 50def _SrcRelative(path): 51 """Returns relative path to top-level src dir.""" 52 return os.path.relpath(path, build_utils.DIR_SOURCE_ROOT) 53 54 55def _GenerateProjectFile(android_manifest, 56 android_sdk_root, 57 cache_dir, 58 partials_dir, 59 sources=None, 60 classpath=None, 61 srcjar_sources=None, 62 resource_sources=None, 63 custom_lint_jars=None, 64 custom_annotation_zips=None, 65 android_sdk_version=None, 66 baseline_path=None): 67 project = ElementTree.Element('project') 68 root = ElementTree.SubElement(project, 'root') 69 # Run lint from output directory: crbug.com/1115594 70 root.set('dir', os.getcwd()) 71 sdk = ElementTree.SubElement(project, 'sdk') 72 # Lint requires that the sdk path be an absolute path. 73 sdk.set('dir', os.path.abspath(android_sdk_root)) 74 if baseline_path is not None: 75 baseline = ElementTree.SubElement(project, 'baseline') 76 baseline.set('file', baseline_path) 77 cache = ElementTree.SubElement(project, 'cache') 78 cache.set('dir', cache_dir) 79 main_module = ElementTree.SubElement(project, 'module') 80 main_module.set('name', 'main') 81 main_module.set('android', 'true') 82 main_module.set('library', 'false') 83 # Required to make lint-resources.xml be written to a per-target path. 84 # https://crbug.com/1515070 and b/324598620 85 main_module.set('partial-results-dir', partials_dir) 86 if android_sdk_version: 87 main_module.set('compile_sdk_version', android_sdk_version) 88 manifest = ElementTree.SubElement(main_module, 'manifest') 89 manifest.set('file', android_manifest) 90 if srcjar_sources: 91 for srcjar_file in srcjar_sources: 92 src = ElementTree.SubElement(main_module, 'src') 93 src.set('file', srcjar_file) 94 # Cannot add generated="true" since then lint does not scan them, and 95 # we get "UnusedResources" lint errors when resources are used only by 96 # generated files. 97 if sources: 98 for source in sources: 99 src = ElementTree.SubElement(main_module, 'src') 100 src.set('file', source) 101 # Cannot set test="true" since we sometimes put Test.java files beside 102 # non-test files, which lint does not allow: 103 # "Test sources cannot be in the same source root as production files" 104 if classpath: 105 for file_path in classpath: 106 classpath_element = ElementTree.SubElement(main_module, 'classpath') 107 classpath_element.set('file', file_path) 108 if resource_sources: 109 for resource_file in resource_sources: 110 resource = ElementTree.SubElement(main_module, 'resource') 111 resource.set('file', resource_file) 112 if custom_lint_jars: 113 for lint_jar in custom_lint_jars: 114 lint = ElementTree.SubElement(main_module, 'lint-checks') 115 lint.set('file', lint_jar) 116 if custom_annotation_zips: 117 for annotation_zip in custom_annotation_zips: 118 annotation = ElementTree.SubElement(main_module, 'annotations') 119 annotation.set('file', annotation_zip) 120 return project 121 122 123def _RetrieveBackportedMethods(backported_methods_path): 124 with open(backported_methods_path) as f: 125 methods = f.read().splitlines() 126 # Methods look like: 127 # java/util/Set#of(Ljava/lang/Object;)Ljava/util/Set; 128 # But error message looks like: 129 # Call requires API level R (current min is 21): java.util.Set#of [NewApi] 130 methods = (m.replace('/', '\\.') for m in methods) 131 methods = (m[:m.index('(')] for m in methods) 132 return sorted(set(methods)) 133 134 135def _GenerateConfigXmlTree(orig_config_path, backported_methods): 136 if orig_config_path: 137 root_node = ElementTree.parse(orig_config_path).getroot() 138 else: 139 root_node = ElementTree.fromstring('<lint/>') 140 141 issue_node = ElementTree.SubElement(root_node, 'issue') 142 issue_node.attrib['id'] = 'NewApi' 143 ignore_node = ElementTree.SubElement(issue_node, 'ignore') 144 ignore_node.attrib['regexp'] = '|'.join(backported_methods) 145 return root_node 146 147 148def _GenerateAndroidManifest(original_manifest_path, extra_manifest_paths, 149 min_sdk_version, android_sdk_version): 150 # Set minSdkVersion in the manifest to the correct value. 151 doc, manifest, app_node = manifest_utils.ParseManifest(original_manifest_path) 152 153 # TODO(crbug.com/40148088): Should this be done using manifest merging? 154 # Add anything in the application node of the extra manifests to the main 155 # manifest to prevent unused resource errors. 156 for path in extra_manifest_paths: 157 _, _, extra_app_node = manifest_utils.ParseManifest(path) 158 for node in extra_app_node: 159 app_node.append(node) 160 161 uses_sdk = manifest.find('./uses-sdk') 162 if uses_sdk is None: 163 uses_sdk = ElementTree.Element('uses-sdk') 164 manifest.insert(0, uses_sdk) 165 uses_sdk.set('{%s}minSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 166 min_sdk_version) 167 uses_sdk.set('{%s}targetSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 168 android_sdk_version) 169 return doc 170 171 172def _WriteXmlFile(root, path): 173 logging.info('Writing xml file %s', path) 174 build_utils.MakeDirectory(os.path.dirname(path)) 175 with action_helpers.atomic_output(path) as f: 176 # Although we can write it just with ElementTree.tostring, using minidom 177 # makes it a lot easier to read as a human (also on code search). 178 f.write( 179 minidom.parseString(ElementTree.tostring( 180 root, encoding='utf-8')).toprettyxml(indent=' ').encode('utf-8')) 181 182 183def _RunLint(custom_lint_jar_path, 184 lint_jar_path, 185 backported_methods_path, 186 config_path, 187 manifest_path, 188 extra_manifest_paths, 189 sources, 190 classpath, 191 cache_dir, 192 android_sdk_version, 193 aars, 194 srcjars, 195 min_sdk_version, 196 resource_sources, 197 resource_zips, 198 android_sdk_root, 199 lint_gen_dir, 200 baseline, 201 warnings_as_errors=False): 202 logging.info('Lint starting') 203 if not cache_dir: 204 # Use per-target cache directory when --cache-dir is not used. 205 cache_dir = os.path.join(lint_gen_dir, 'cache') 206 # Lint complains if the directory does not exist. 207 # When --create-cache is used, ninja will create this directory because the 208 # stamp file is created within it. 209 os.makedirs(cache_dir, exist_ok=True) 210 211 if baseline and not os.path.exists(baseline): 212 # Generating new baselines is only done locally, and requires more memory to 213 # avoid OOMs. 214 creating_baseline = True 215 lint_xmx = '4G' 216 else: 217 creating_baseline = False 218 lint_xmx = '2G' 219 220 # Lint requires this directory to exist and be cleared. 221 # See b/324598620 222 partials_dir = os.path.join(lint_gen_dir, 'partials') 223 shutil.rmtree(partials_dir, ignore_errors=True) 224 os.makedirs(partials_dir) 225 226 # All paths in lint are based off of relative paths from root with root as the 227 # prefix. Path variable substitution is based off of prefix matching so custom 228 # path variables need to match exactly in order to show up in baseline files. 229 # e.g. lint_path=path/to/output/dir/../../file/in/src 230 root_path = os.getcwd() # This is usually the output directory. 231 pathvar_src = os.path.join( 232 root_path, os.path.relpath(build_utils.DIR_SOURCE_ROOT, start=root_path)) 233 234 cmd = build_utils.JavaCmd(xmx=lint_xmx) + [ 235 '-cp', 236 '{}:{}'.format(lint_jar_path, custom_lint_jar_path), 237 'org.chromium.build.CustomLint', 238 '--sdk-home', 239 android_sdk_root, 240 '--jdk-home', 241 build_utils.JAVA_HOME, 242 '--path-variables', 243 f'SRC={pathvar_src}', 244 '--offline', 245 '--quiet', # Silences lint's "." progress updates. 246 '--stacktrace', # Prints full stacktraces for internal lint errors. 247 '--disable', 248 ','.join(_DISABLED_ALWAYS), 249 ] 250 251 if not manifest_path: 252 manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build', 253 'android', 'AndroidManifest.xml') 254 255 logging.info('Generating config.xml') 256 backported_methods = _RetrieveBackportedMethods(backported_methods_path) 257 config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods) 258 generated_config_path = os.path.join(lint_gen_dir, 'config.xml') 259 _WriteXmlFile(config_xml_node, generated_config_path) 260 cmd.extend(['--config', generated_config_path]) 261 262 logging.info('Generating Android manifest file') 263 android_manifest_tree = _GenerateAndroidManifest(manifest_path, 264 extra_manifest_paths, 265 min_sdk_version, 266 android_sdk_version) 267 # Just use a hardcoded name, since we may have different target names (and 268 # thus different manifest_paths) using the same lint baseline. Eg. 269 # trichrome_chrome_bundle and trichrome_chrome_32_64_bundle. 270 lint_android_manifest_path = os.path.join(lint_gen_dir, 'AndroidManifest.xml') 271 _WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path) 272 273 resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR) 274 # These are zip files with generated resources (e. g. strings from GRD). 275 logging.info('Extracting resource zips') 276 for resource_zip in resource_zips: 277 # Use a consistent root and name rather than a temporary file so that 278 # suppressions can be local to the lint target and the resource target. 279 resource_dir = os.path.join(resource_root_dir, resource_zip) 280 shutil.rmtree(resource_dir, True) 281 os.makedirs(resource_dir) 282 resource_sources.extend( 283 build_utils.ExtractAll(resource_zip, path=resource_dir)) 284 285 logging.info('Extracting aars') 286 aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR) 287 custom_lint_jars = [] 288 custom_annotation_zips = [] 289 if aars: 290 for aar in aars: 291 # Use relative source for aar files since they are not generated. 292 aar_dir = os.path.join(aar_root_dir, 293 os.path.splitext(_SrcRelative(aar))[0]) 294 shutil.rmtree(aar_dir, True) 295 os.makedirs(aar_dir) 296 aar_files = build_utils.ExtractAll(aar, path=aar_dir) 297 for f in aar_files: 298 if f.endswith('lint.jar'): 299 custom_lint_jars.append(f) 300 elif f.endswith('annotations.zip'): 301 custom_annotation_zips.append(f) 302 303 logging.info('Extracting srcjars') 304 srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR) 305 srcjar_sources = [] 306 if srcjars: 307 for srcjar in srcjars: 308 # Use path without extensions since otherwise the file name includes 309 # .srcjar and lint treats it as a srcjar. 310 srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0]) 311 shutil.rmtree(srcjar_dir, True) 312 os.makedirs(srcjar_dir) 313 # Sadly lint's srcjar support is broken since it only considers the first 314 # srcjar. Until we roll a lint version with that fixed, we need to extract 315 # it ourselves. 316 srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir)) 317 318 logging.info('Generating project file') 319 project_file_root = _GenerateProjectFile( 320 lint_android_manifest_path, android_sdk_root, cache_dir, partials_dir, 321 sources, classpath, srcjar_sources, resource_sources, custom_lint_jars, 322 custom_annotation_zips, android_sdk_version, baseline) 323 324 project_xml_path = os.path.join(lint_gen_dir, 'project.xml') 325 _WriteXmlFile(project_file_root, project_xml_path) 326 cmd += ['--project', project_xml_path] 327 328 # This filter is necessary for JDK11. 329 stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings 330 stdout_filter = lambda x: build_utils.FilterLines(x, 'No issues found') 331 332 start = time.time() 333 logging.debug('Lint command %s', ' '.join(cmd)) 334 failed = False 335 336 if creating_baseline and not warnings_as_errors: 337 # Allow error code 6 when creating a baseline: ERRNO_CREATED_BASELINE 338 fail_func = lambda returncode, _: returncode not in (0, 6) 339 else: 340 fail_func = lambda returncode, _: returncode != 0 341 342 try: 343 build_utils.CheckOutput(cmd, 344 print_stdout=True, 345 stdout_filter=stdout_filter, 346 stderr_filter=stderr_filter, 347 fail_on_output=warnings_as_errors, 348 fail_func=fail_func) 349 except build_utils.CalledProcessError as e: 350 failed = True 351 # Do not output the python stacktrace because it is lengthy and is not 352 # relevant to the actual lint error. 353 sys.stderr.write(e.output) 354 finally: 355 # When not treating warnings as errors, display the extra footer. 356 is_debug = os.environ.get('LINT_DEBUG', '0') != '0' 357 358 end = time.time() - start 359 logging.info('Lint command took %ss', end) 360 if not is_debug: 361 shutil.rmtree(aar_root_dir, ignore_errors=True) 362 shutil.rmtree(resource_root_dir, ignore_errors=True) 363 shutil.rmtree(srcjar_root_dir, ignore_errors=True) 364 os.unlink(project_xml_path) 365 shutil.rmtree(partials_dir, ignore_errors=True) 366 367 if failed: 368 print('- For more help with lint in Chrome:', _LINT_MD_URL) 369 if is_debug: 370 print('- DEBUG MODE: Here is the project.xml: {}'.format( 371 _SrcRelative(project_xml_path))) 372 else: 373 print('- Run with LINT_DEBUG=1 to enable lint configuration debugging') 374 sys.exit(1) 375 376 logging.info('Lint completed') 377 378 379def _ParseArgs(argv): 380 parser = argparse.ArgumentParser() 381 action_helpers.add_depfile_arg(parser) 382 parser.add_argument('--target-name', help='Fully qualified GN target name.') 383 parser.add_argument('--skip-build-server', 384 action='store_true', 385 help='Avoid using the build server.') 386 parser.add_argument('--use-build-server', 387 action='store_true', 388 help='Always use the build server.') 389 parser.add_argument('--lint-jar-path', 390 required=True, 391 help='Path to the lint jar.') 392 parser.add_argument('--custom-lint-jar-path', 393 required=True, 394 help='Path to our custom lint jar.') 395 parser.add_argument('--backported-methods', 396 help='Path to backported methods file created by R8.') 397 parser.add_argument('--cache-dir', 398 help='Path to the directory in which the android cache ' 399 'directory tree should be stored.') 400 parser.add_argument('--config-path', help='Path to lint suppressions file.') 401 parser.add_argument('--lint-gen-dir', 402 required=True, 403 help='Path to store generated xml files.') 404 parser.add_argument('--stamp', help='Path to stamp upon success.') 405 parser.add_argument('--android-sdk-version', 406 help='Version (API level) of the Android SDK used for ' 407 'building.') 408 parser.add_argument('--min-sdk-version', 409 required=True, 410 help='Minimal SDK version to lint against.') 411 parser.add_argument('--android-sdk-root', 412 required=True, 413 help='Lint needs an explicit path to the android sdk.') 414 parser.add_argument('--create-cache', 415 action='store_true', 416 help='Whether this invocation is just warming the cache.') 417 parser.add_argument('--warnings-as-errors', 418 action='store_true', 419 help='Treat all warnings as errors.') 420 parser.add_argument('--sources', 421 help='A list of files containing java and kotlin source ' 422 'files.') 423 parser.add_argument('--aars', help='GN list of included aars.') 424 parser.add_argument('--srcjars', help='GN list of included srcjars.') 425 parser.add_argument('--manifest-path', 426 help='Path to original AndroidManifest.xml') 427 parser.add_argument('--extra-manifest-paths', 428 action='append', 429 help='GYP-list of manifest paths to merge into the ' 430 'original AndroidManifest.xml') 431 parser.add_argument('--resource-sources', 432 default=[], 433 action='append', 434 help='GYP-list of resource sources files, similar to ' 435 'java sources files, but for resource files.') 436 parser.add_argument('--resource-zips', 437 default=[], 438 action='append', 439 help='GYP-list of resource zips, zip files of generated ' 440 'resource files.') 441 parser.add_argument('--classpath', 442 help='List of jars to add to the classpath.') 443 parser.add_argument('--baseline', 444 help='Baseline file to ignore existing errors and fail ' 445 'on new errors.') 446 447 args = parser.parse_args(build_utils.ExpandFileArgs(argv)) 448 args.sources = action_helpers.parse_gn_list(args.sources) 449 args.aars = action_helpers.parse_gn_list(args.aars) 450 args.srcjars = action_helpers.parse_gn_list(args.srcjars) 451 args.resource_sources = action_helpers.parse_gn_list(args.resource_sources) 452 args.extra_manifest_paths = action_helpers.parse_gn_list( 453 args.extra_manifest_paths) 454 args.resource_zips = action_helpers.parse_gn_list(args.resource_zips) 455 args.classpath = action_helpers.parse_gn_list(args.classpath) 456 457 if args.baseline: 458 assert os.path.basename(args.baseline) == 'lint-baseline.xml', ( 459 'The baseline file needs to be named "lint-baseline.xml" in order for ' 460 'the autoroller to find and update it whenever lint is rolled to a new ' 461 'version.') 462 463 return args 464 465 466def main(): 467 build_utils.InitLogging('LINT_DEBUG') 468 args = _ParseArgs(sys.argv[1:]) 469 470 sources = [] 471 for sources_file in args.sources: 472 sources.extend(build_utils.ReadSourcesList(sources_file)) 473 resource_sources = [] 474 for resource_sources_file in args.resource_sources: 475 resource_sources.extend(build_utils.ReadSourcesList(resource_sources_file)) 476 477 possible_depfile_deps = (args.srcjars + args.resource_zips + sources + 478 resource_sources + [ 479 args.baseline, 480 args.manifest_path, 481 ]) 482 depfile_deps = [p for p in possible_depfile_deps if p] 483 484 if args.depfile: 485 action_helpers.write_depfile(args.depfile, args.stamp, depfile_deps) 486 487 # TODO(wnwen): Consider removing lint cache now that there are only two lint 488 # invocations. 489 # Avoid parallelizing cache creation since lint runs without the cache defeat 490 # the purpose of creating the cache in the first place. Forward the command 491 # after the depfile has been written as siso requires it. 492 if (not args.create_cache and not args.skip_build_server 493 and server_utils.MaybeRunCommand(name=args.target_name, 494 argv=sys.argv, 495 stamp_file=args.stamp, 496 force=args.use_build_server)): 497 return 498 499 _RunLint(args.custom_lint_jar_path, 500 args.lint_jar_path, 501 args.backported_methods, 502 args.config_path, 503 args.manifest_path, 504 args.extra_manifest_paths, 505 sources, 506 args.classpath, 507 args.cache_dir, 508 args.android_sdk_version, 509 args.aars, 510 args.srcjars, 511 args.min_sdk_version, 512 resource_sources, 513 args.resource_zips, 514 args.android_sdk_root, 515 args.lint_gen_dir, 516 args.baseline, 517 warnings_as_errors=args.warnings_as_errors) 518 logging.info('Creating stamp file') 519 build_utils.Touch(args.stamp) 520 521 522if __name__ == '__main__': 523 sys.exit(main()) 524