1#!/usr/bin/env python3 2# Copyright 2017 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Checks for incomplete direct deps.""" 6 7import argparse 8import collections 9import logging 10import pathlib 11import shlex 12import sys 13from typing import Dict, List, Optional, Set, Tuple 14 15from util import build_utils 16from util import dep_utils 17from util import jar_utils 18from util import server_utils 19import action_helpers # build_utils adds //build to sys.path. 20 21_SRC_PATH = pathlib.Path(build_utils.DIR_SOURCE_ROOT).resolve() 22sys.path.insert(1, str(_SRC_PATH / 'build/gn_ast')) 23from gn_editor import NO_VALID_GN_STR 24 25 26def _ShouldIgnoreDep(dep_name: str): 27 if 'gen.base_module.R' in dep_name: 28 return True 29 return False 30 31 32def _ParseDepGraph(jar_path: str): 33 output = jar_utils.run_jdeps(pathlib.Path(jar_path)) 34 assert output is not None, f'Unable to parse jdep for {jar_path}' 35 dep_graph = collections.defaultdict(set) 36 # pylint: disable=line-too-long 37 # Example output: 38 # java.javac.jar -> java.base 39 # java.javac.jar -> not found 40 # org.chromium.chrome.browser.tabmodel.AsyncTabParamsManagerFactory -> java.lang.Object java.base 41 # org.chromium.chrome.browser.tabmodel.TabWindowManagerImpl -> org.chromium.base.ApplicationStatus not found 42 # org.chromium.chrome.browser.tabmodel.TabWindowManagerImpl -> org.chromium.base.ApplicationStatus$ActivityStateListener not found 43 # org.chromium.chrome.browser.tabmodel.TabWindowManagerImpl -> org.chromium.chrome.browser.tab.Tab not found 44 # pylint: enable=line-too-long 45 for line in output.splitlines(): 46 parsed = line.split() 47 # E.g. java.javac.jar -> java.base 48 if len(parsed) <= 3: 49 continue 50 # E.g. java.javac.jar -> not found 51 if parsed[2] == 'not' and parsed[3] == 'found': 52 continue 53 if parsed[1] != '->': 54 continue 55 dep_from = parsed[0] 56 dep_to = parsed[2] 57 dep_graph[dep_from].add(dep_to) 58 return dep_graph 59 60 61def _EnsureDirectClasspathIsComplete( 62 *, 63 input_jar: str, 64 gn_target: str, 65 output_dir: str, 66 sdk_classpath_jars: List[str], 67 direct_classpath_jars: List[str], 68 full_classpath_jars: List[str], 69 full_classpath_gn_targets: List[str], 70 warnings_as_errors: bool, 71 auto_add_deps: bool, 72): 73 logging.info('Parsing %d direct classpath jars', len(sdk_classpath_jars)) 74 sdk_classpath_deps = set() 75 for jar in sdk_classpath_jars: 76 deps = jar_utils.extract_full_class_names_from_jar(jar) 77 sdk_classpath_deps.update(deps) 78 79 logging.info('Parsing %d direct classpath jars', len(direct_classpath_jars)) 80 direct_classpath_deps = set() 81 for jar in direct_classpath_jars: 82 deps = jar_utils.extract_full_class_names_from_jar(jar) 83 direct_classpath_deps.update(deps) 84 85 logging.info('Parsing %d full classpath jars', len(full_classpath_jars)) 86 full_classpath_deps = set() 87 dep_to_target = collections.defaultdict(set) 88 for jar, target in zip(full_classpath_jars, full_classpath_gn_targets): 89 deps = jar_utils.extract_full_class_names_from_jar(jar) 90 full_classpath_deps.update(deps) 91 for dep in deps: 92 dep_to_target[dep].add(target) 93 94 transitive_deps = full_classpath_deps - direct_classpath_deps 95 96 missing_class_to_caller: Dict[str, str] = {} 97 dep_graph = _ParseDepGraph(input_jar) 98 logging.info('Finding missing deps from %d classes', len(dep_graph)) 99 # dep_graph.keys() is a list of all the classes in the current input_jar. Skip 100 # all of these to avoid checking dependencies in the same target (e.g. A 101 # depends on B, but both A and B are in input_jar). 102 # Since the bundle will always have access to classes in the current android 103 # sdk, those should not be considered missing. 104 seen_deps = set(dep_graph.keys()) | sdk_classpath_deps 105 for dep_from, deps_to in dep_graph.items(): 106 for dep_to in deps_to - seen_deps: 107 if _ShouldIgnoreDep(dep_to): 108 continue 109 seen_deps.add(dep_to) 110 if dep_to in transitive_deps: 111 # Allow clobbering since it doesn't matter which specific class depends 112 # on |dep_to|. 113 missing_class_to_caller[dep_to] = dep_from 114 115 if missing_class_to_caller: 116 _ProcessMissingDirectClasspathDeps(missing_class_to_caller, dep_to_target, 117 gn_target, output_dir, 118 warnings_as_errors, auto_add_deps) 119 120 121def _ProcessMissingDirectClasspathDeps( 122 missing_class_to_caller: Dict[str, str], 123 dep_to_target: Dict[str, Set[str]], 124 gn_target: str, 125 output_dir: str, 126 warnings_as_errors: bool, 127 auto_add_deps: bool, 128): 129 potential_targets_to_missing_classes: Dict[ 130 Tuple, List[str]] = collections.defaultdict(list) 131 for missing_class in missing_class_to_caller: 132 potential_targets = tuple(sorted(dep_to_target[missing_class])) 133 potential_targets_to_missing_classes[potential_targets].append( 134 missing_class) 135 136 deps_to_add_programatically = _DisambiguateMissingDeps( 137 potential_targets_to_missing_classes, output_dir=output_dir) 138 139 cmd = dep_utils.CreateAddDepsCommand(gn_target, 140 sorted(deps_to_add_programatically)) 141 142 if not auto_add_deps: 143 _PrintAndMaybeExit(potential_targets_to_missing_classes, 144 missing_class_to_caller, gn_target, warnings_as_errors, 145 cmd) 146 else: 147 failed = False 148 try: 149 stdout = build_utils.CheckOutput(cmd, 150 cwd=build_utils.DIR_SOURCE_ROOT, 151 fail_on_output=warnings_as_errors) 152 if f'Unable to find {gn_target}' in stdout: 153 # This can happen if a target's deps are stored in a variable instead 154 # of a list and then simply assigned: `deps = deps_variable`. These 155 # need to be manually added to the `deps_variable`. 156 failed = True 157 except build_utils.CalledProcessError as e: 158 if NO_VALID_GN_STR in e.output: 159 failed = True 160 else: 161 raise 162 163 build_file_path = dep_utils.GnTargetToBuildFilePath(gn_target) 164 if failed: 165 print(f'Unable to auto-add missing dep(s) to {build_file_path}.') 166 _PrintAndMaybeExit(potential_targets_to_missing_classes, 167 missing_class_to_caller, gn_target, warnings_as_errors) 168 else: 169 gn_target_name = gn_target.split(':', 1)[-1] 170 print(f'Successfully updated "{gn_target_name}" in {build_file_path} ' 171 f'with missing direct deps: {deps_to_add_programatically}') 172 173 174def _DisambiguateMissingDeps( 175 potential_targets_to_missing_classes: Dict[Tuple, List[str]], 176 output_dir: str, 177): 178 deps_to_add_programatically = set() 179 class_lookup_index = None 180 # Run this even when len(potential_targets) == 1, because we need to look for 181 # java_group()s with preferred_deps=true. 182 for (potential_targets, 183 missing_classes) in potential_targets_to_missing_classes.items(): 184 # Dict for ordered dict. 185 potential_targets = {t: True for t in potential_targets} 186 # Rather than just picking any of the potential targets, we want to use 187 # dep_utils.ClassLookupIndex to ensure we respect the preferred dep if any 188 # exists for the missing deps. It is necessary to obtain the preferred dep 189 # status of these potential targets by matching them to a ClassEntry. 190 target_name_to_class_entry: Dict[str, dep_utils.ClassEntry] = {} 191 for missing_class in missing_classes: 192 # Lazily create the ClassLookupIndex in case all potential_targets lists 193 # are only 1 element in length. 194 if class_lookup_index is None: 195 class_lookup_index = dep_utils.ClassLookupIndex( 196 pathlib.Path(output_dir), should_build=False) 197 for class_entry in class_lookup_index.match(missing_class): 198 target_name_to_class_entry[class_entry.target] = class_entry 199 # Ensure preferred deps are always included in the options. 200 if (class_entry.preferred_dep 201 and class_entry.target not in potential_targets): 202 potential_targets[class_entry.target] = True 203 potential_class_entries = [ 204 target_name_to_class_entry[t] for t in potential_targets 205 ] 206 potential_class_entries = dep_utils.DisambiguateDeps( 207 potential_class_entries) 208 deps_to_add_programatically.add(potential_class_entries[0].target) 209 return deps_to_add_programatically 210 211 212def _PrintAndMaybeExit( 213 potential_targets_to_missing_classes: Dict[Tuple, List[str]], 214 missing_class_to_caller: Dict[str, str], 215 gn_target: str, 216 warnings_as_errors: bool, 217 cmd: Optional[List[str]] = None, 218): 219 print('=' * 30 + ' Dependency Checks Failed ' + '=' * 30) 220 print(f'Target: {gn_target}') 221 print('Direct classpath is incomplete. To fix, add deps on:') 222 for (potential_targets, 223 missing_classes) in potential_targets_to_missing_classes.items(): 224 if len(potential_targets) == 1: 225 print(f' * {potential_targets[0]}') 226 else: 227 print(f' * One of {", ".join(potential_targets)}') 228 for missing_class in missing_classes: 229 caller = missing_class_to_caller[missing_class] 230 print(f' ** {missing_class} (needed by {caller})') 231 if cmd: 232 print('\nHint: Run the following command to add the missing deps:') 233 print(f' {shlex.join(cmd)}\n') 234 if warnings_as_errors: 235 sys.exit(1) 236 237 238def main(argv): 239 build_utils.InitLogging('MISSING_DEPS_DEBUG') 240 argv = build_utils.ExpandFileArgs(argv[1:]) 241 parser = argparse.ArgumentParser() 242 parser.add_argument('--target-name', help='Fully qualified GN target name.') 243 parser.add_argument('--use-build-server', 244 action='store_true', 245 help='Always use the build server.') 246 parser.add_argument('--gn-target', required=True) 247 parser.add_argument('--input-jar', required=True) 248 parser.add_argument('--direct-classpath-jars') 249 parser.add_argument('--sdk-classpath-jars') 250 parser.add_argument('--full-classpath-jars') 251 parser.add_argument('--full-classpath-gn-targets') 252 parser.add_argument('--chromium-output-dir') 253 parser.add_argument('--depfile') 254 parser.add_argument('--stamp') 255 parser.add_argument('--warnings-as-errors', 256 action='store_true', 257 help='Treat all warnings as errors.') 258 parser.add_argument( 259 '--auto-add-deps', 260 action='store_true', 261 help='Attempt to automatically add missing deps to the corresponding ' 262 'BUILD.gn file.') 263 args = parser.parse_args(argv) 264 265 args.sdk_classpath_jars = action_helpers.parse_gn_list( 266 args.sdk_classpath_jars) 267 args.direct_classpath_jars = action_helpers.parse_gn_list( 268 args.direct_classpath_jars) 269 args.full_classpath_jars = action_helpers.parse_gn_list( 270 args.full_classpath_jars) 271 args.full_classpath_gn_targets = [ 272 dep_utils.ReplaceGmsPackageIfNeeded(t) 273 for t in action_helpers.parse_gn_list(args.full_classpath_gn_targets) 274 ] 275 276 # No need to rebuild if full_classpath_jars change and direct_classpath_jars 277 # do not. 278 depfile_deps = args.direct_classpath_jars + args.sdk_classpath_jars 279 action_helpers.write_depfile(args.depfile, args.stamp, depfile_deps) 280 281 if server_utils.MaybeRunCommand(name=args.target_name, 282 argv=sys.argv, 283 stamp_file=args.stamp, 284 use_build_server=args.use_build_server): 285 return 286 287 logging.info('Processed args for %s, starting direct classpath check.', 288 args.target_name) 289 _EnsureDirectClasspathIsComplete( 290 input_jar=args.input_jar, 291 gn_target=args.gn_target, 292 output_dir=args.chromium_output_dir, 293 sdk_classpath_jars=args.sdk_classpath_jars, 294 direct_classpath_jars=args.direct_classpath_jars, 295 full_classpath_jars=args.full_classpath_jars, 296 full_classpath_gn_targets=args.full_classpath_gn_targets, 297 warnings_as_errors=args.warnings_as_errors, 298 auto_add_deps=args.auto_add_deps) 299 logging.info('Check completed.') 300 301 server_utils.MaybeTouch(args.stamp) 302 303 304if __name__ == '__main__': 305 sys.exit(main(sys.argv)) 306