1#!/usr/bin/env python3 2# Copyright 2017 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Wraps bin/helper/bytecode_processor and expands @FileArgs.""" 7 8import argparse 9import collections 10import logging 11import os 12import pathlib 13import sys 14from typing import Dict, List, Tuple 15 16from util import build_utils 17from util import dep_utils 18from util import jar_utils 19from util import server_utils 20import action_helpers # build_utils adds //build to sys.path. 21 22_SRC_PATH = pathlib.Path(build_utils.DIR_SOURCE_ROOT).resolve() 23sys.path.append(str(_SRC_PATH / 'tools/android/modularization/gn')) 24from dep_operations import NO_VALID_GN_STR 25 26 27# This is a temporary constant to make reverts easier (if necessary). 28# TODO(crbug.com/1099522): Remove this and make jdeps on by default. 29_USE_JDEPS = True 30 31 32def _ShouldIgnoreDep(dep_name: str): 33 if 'gen.base_module.R' in dep_name: 34 return True 35 return False 36 37 38def _ParseDepGraph(jar_path: str, output_dir: str): 39 output = jar_utils.run_jdeps(build_output_dir=pathlib.Path(output_dir), 40 filepath=pathlib.Path(jar_path)) 41 assert output is not None, f'Unable to parse jdep for {jar_path}' 42 dep_graph = collections.defaultdict(set) 43 # pylint: disable=line-too-long 44 # Example output: 45 # java.javac.jar -> java.base 46 # java.javac.jar -> not found 47 # org.chromium.chrome.browser.tabmodel.AsyncTabParamsManagerFactory -> java.lang.Object java.base 48 # org.chromium.chrome.browser.tabmodel.TabWindowManagerImpl -> org.chromium.base.ApplicationStatus not found 49 # org.chromium.chrome.browser.tabmodel.TabWindowManagerImpl -> org.chromium.base.ApplicationStatus$ActivityStateListener not found 50 # org.chromium.chrome.browser.tabmodel.TabWindowManagerImpl -> org.chromium.chrome.browser.tab.Tab not found 51 # pylint: enable=line-too-long 52 for line in output.splitlines(): 53 parsed = line.split() 54 # E.g. java.javac.jar -> java.base 55 if len(parsed) <= 3: 56 continue 57 # E.g. java.javac.jar -> not found 58 if parsed[2] == 'not' and parsed[3] == 'found': 59 continue 60 if parsed[1] != '->': 61 continue 62 dep_from = parsed[0] 63 dep_to = parsed[2] 64 dep_graph[dep_from].add(dep_to) 65 return dep_graph 66 67 68def _GnTargetToBuildFilePath(gn_target: str): 69 """Returns the relative BUILD.gn file path for this target from src root.""" 70 assert gn_target.startswith('//'), f'Relative {gn_target} name not supported.' 71 ninja_target_name = gn_target[2:] 72 73 # Remove the colon at the end 74 colon_index = ninja_target_name.find(':') 75 if colon_index != -1: 76 ninja_target_name = ninja_target_name[:colon_index] 77 78 return os.path.join(ninja_target_name, 'BUILD.gn') 79 80 81def _EnsureDirectClasspathIsComplete( 82 *, 83 input_jar: str, 84 gn_target: str, 85 output_dir: str, 86 sdk_classpath_jars: List[str], 87 direct_classpath_jars: List[str], 88 full_classpath_jars: List[str], 89 full_classpath_gn_targets: List[str], 90 warnings_as_errors: bool, 91 auto_add_deps: bool, 92): 93 logging.info('Parsing %d direct classpath jars', len(sdk_classpath_jars)) 94 sdk_classpath_deps = set() 95 for jar in sdk_classpath_jars: 96 deps = jar_utils.extract_full_class_names_from_jar( 97 build_output_dir=pathlib.Path(output_dir), jar_path=pathlib.Path(jar)) 98 sdk_classpath_deps.update(deps) 99 100 logging.info('Parsing %d direct classpath jars', len(direct_classpath_jars)) 101 direct_classpath_deps = set() 102 for jar in direct_classpath_jars: 103 deps = jar_utils.extract_full_class_names_from_jar( 104 build_output_dir=pathlib.Path(output_dir), jar_path=pathlib.Path(jar)) 105 direct_classpath_deps.update(deps) 106 107 logging.info('Parsing %d full classpath jars', len(full_classpath_jars)) 108 full_classpath_deps = set() 109 dep_to_target = collections.defaultdict(set) 110 for jar, target in zip(full_classpath_jars, full_classpath_gn_targets): 111 deps = jar_utils.extract_full_class_names_from_jar( 112 build_output_dir=pathlib.Path(output_dir), jar_path=pathlib.Path(jar)) 113 full_classpath_deps.update(deps) 114 for dep in deps: 115 dep_to_target[dep].add(target) 116 117 transitive_deps = full_classpath_deps - direct_classpath_deps 118 119 missing_classes: Dict[str, str] = {} 120 dep_graph = _ParseDepGraph(input_jar, output_dir) 121 logging.info('Finding missing deps from %d classes', len(dep_graph)) 122 # dep_graph.keys() is a list of all the classes in the current input_jar. Skip 123 # all of these to avoid checking dependencies in the same target (e.g. A 124 # depends on B, but both A and B are in input_jar). 125 # Since the bundle will always have access to classes in the current android 126 # sdk, those should not be considered missing. 127 seen_deps = set(dep_graph.keys()) | sdk_classpath_deps 128 for dep_from, deps_to in dep_graph.items(): 129 for dep_to in deps_to - seen_deps: 130 if _ShouldIgnoreDep(dep_to): 131 continue 132 seen_deps.add(dep_to) 133 if dep_to in transitive_deps: 134 # Allow clobbering since it doesn't matter which specific class depends 135 # on |dep_to|. 136 missing_classes[dep_to] = dep_from 137 138 # missing_target_names = tuple(sorted(dep_to_target[dep_to])) 139 # missing_targets[missing_target_names][dep_to] = dep_from 140 if missing_classes: 141 142 def print_and_maybe_exit(): 143 missing_targets: Dict[Tuple, List[str]] = collections.defaultdict(list) 144 for dep_to, dep_from in missing_classes.items(): 145 missing_target_names = tuple(sorted(dep_to_target[dep_to])) 146 missing_targets[missing_target_names].append(dep_to) 147 print('=' * 30 + ' Dependency Checks Failed ' + '=' * 30) 148 print(f'Target: {gn_target}') 149 print('Direct classpath is incomplete. To fix, add deps on:') 150 for missing_target_names, deps_to in missing_targets.items(): 151 if len(missing_target_names) > 1: 152 print(f' * One of {", ".join(missing_target_names)}') 153 else: 154 print(f' * {missing_target_names[0]}') 155 for dep_to in deps_to: 156 dep_from = missing_classes[dep_to] 157 print(f' ** {dep_to} (needed by {dep_from})') 158 if warnings_as_errors: 159 sys.exit(1) 160 161 if not auto_add_deps: 162 print_and_maybe_exit() 163 else: 164 # Normalize chrome_public_apk__java to chrome_public_apk. 165 gn_target = gn_target.split('__', 1)[0] 166 167 # TODO(https://crbug.com/1099522): This should be generalized into util. 168 build_file_path = _GnTargetToBuildFilePath(gn_target) 169 cmd = [ 170 'tools/android/modularization/gn/dep_operations.py', 'add', '--quiet', 171 '--file', build_file_path, '--target', gn_target, '--deps' 172 ] 173 class_lookup_index = dep_utils.ClassLookupIndex(pathlib.Path(output_dir), 174 should_build=False) 175 176 missing_deps = set() 177 for dep_to in missing_classes: 178 # Using dep_utils.ClassLookupIndex ensures we respect the preferred dep 179 # if any exists for the missing deps. 180 suggested_deps = class_lookup_index.match(dep_to) 181 assert suggested_deps, f'Unable to find target for {dep_to}' 182 suggested_deps = dep_utils.DisambiguateDeps(suggested_deps) 183 missing_deps.add(suggested_deps[0].target) 184 cmd += missing_deps 185 failed = False 186 187 try: 188 stdout = build_utils.CheckOutput(cmd, 189 cwd=build_utils.DIR_SOURCE_ROOT, 190 fail_on_output=warnings_as_errors) 191 if f'Unable to find {gn_target}' in stdout: 192 # This can happen if a target's deps are stored in a variable instead 193 # of a list and then simply assigned: `deps = deps_variable`. These 194 # need to be manually added to the `deps_variable`. 195 failed = True 196 except build_utils.CalledProcessError as e: 197 if NO_VALID_GN_STR in e.output: 198 failed = True 199 else: 200 raise 201 if failed: 202 print(f'Unable to auto-add missing dep(s) to {build_file_path}.') 203 print_and_maybe_exit() 204 else: 205 gn_target_name = gn_target.split(':', 1)[-1] 206 print(f'Successfully updated "{gn_target_name}" in {build_file_path} ' 207 f'with missing direct deps: {missing_deps}') 208 209 210def main(argv): 211 build_utils.InitLogging('BYTECODE_PROCESSOR_DEBUG') 212 argv = build_utils.ExpandFileArgs(argv[1:]) 213 parser = argparse.ArgumentParser() 214 parser.add_argument('--target-name', help='Fully qualified GN target name.') 215 parser.add_argument('--use-build-server', 216 action='store_true', 217 help='Always use the build server.') 218 parser.add_argument('--script', required=True, 219 help='Path to the java binary wrapper script.') 220 parser.add_argument('--gn-target', required=True) 221 parser.add_argument('--input-jar', required=True) 222 parser.add_argument('--direct-classpath-jars') 223 parser.add_argument('--sdk-classpath-jars') 224 parser.add_argument('--full-classpath-jars') 225 parser.add_argument('--full-classpath-gn-targets') 226 parser.add_argument('--chromium-output-dir') 227 parser.add_argument('--stamp') 228 parser.add_argument('-v', '--verbose', action='store_true') 229 parser.add_argument('--missing-classes-allowlist') 230 parser.add_argument('--warnings-as-errors', 231 action='store_true', 232 help='Treat all warnings as errors.') 233 parser.add_argument( 234 '--auto-add-deps', 235 action='store_true', 236 help='Attempt to automatically add missing deps to the corresponding ' 237 'BUILD.gn file.') 238 args = parser.parse_args(argv) 239 240 if server_utils.MaybeRunCommand(name=args.target_name, 241 argv=sys.argv, 242 stamp_file=args.stamp, 243 force=args.use_build_server): 244 return 245 246 args.sdk_classpath_jars = action_helpers.parse_gn_list( 247 args.sdk_classpath_jars) 248 args.direct_classpath_jars = action_helpers.parse_gn_list( 249 args.direct_classpath_jars) 250 args.full_classpath_jars = action_helpers.parse_gn_list( 251 args.full_classpath_jars) 252 args.full_classpath_gn_targets = [ 253 dep_utils.ReplaceGmsPackageIfNeeded(t) 254 for t in action_helpers.parse_gn_list(args.full_classpath_gn_targets) 255 ] 256 args.missing_classes_allowlist = action_helpers.parse_gn_list( 257 args.missing_classes_allowlist) 258 259 verbose = '--verbose' if args.verbose else '--not-verbose' 260 261 # TODO(https://crbug.com/1099522): Make jdeps the default. 262 if _USE_JDEPS: 263 logging.info('Processed args for %s, starting direct classpath check.', 264 args.target_name) 265 _EnsureDirectClasspathIsComplete( 266 input_jar=args.input_jar, 267 gn_target=args.gn_target, 268 output_dir=args.chromium_output_dir, 269 sdk_classpath_jars=args.sdk_classpath_jars, 270 direct_classpath_jars=args.direct_classpath_jars, 271 full_classpath_jars=args.full_classpath_jars, 272 full_classpath_gn_targets=args.full_classpath_gn_targets, 273 warnings_as_errors=args.warnings_as_errors, 274 auto_add_deps=args.auto_add_deps, 275 ) 276 logging.info('Check completed.') 277 else: 278 cmd = [ 279 args.script, args.gn_target, args.input_jar, verbose, '--not-prebuilt' 280 ] 281 cmd += [str(len(args.missing_classes_allowlist))] 282 cmd += args.missing_classes_allowlist 283 cmd += [str(len(args.sdk_classpath_jars))] 284 cmd += args.sdk_classpath_jars 285 cmd += [str(len(args.direct_classpath_jars))] 286 cmd += args.direct_classpath_jars 287 cmd += [str(len(args.full_classpath_jars))] 288 cmd += args.full_classpath_jars 289 cmd += [str(len(args.full_classpath_gn_targets))] 290 cmd += args.full_classpath_gn_targets 291 try: 292 build_utils.CheckOutput( 293 cmd, 294 print_stdout=True, 295 fail_func=None, # type: ignore 296 fail_on_output=args.warnings_as_errors) 297 except build_utils.CalledProcessError as e: 298 # Do not output command line because it is massive and makes the actual 299 # error message hard to find. 300 sys.stderr.write(e.output) 301 sys.exit(1) 302 303 if args.stamp: 304 build_utils.Touch(args.stamp) 305 306 307if __name__ == '__main__': 308 sys.exit(main(sys.argv)) 309