1#!/usr/bin/env python3 2# 3# Copyright (C) 2021 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""A json-module-graph postprocessing script to generate a bp2build progress tracker. 17 18Usage: 19 ./bp2build-progress.py [report|graph] -m <module name> 20 21Example: 22 23 To generate a report on the `adbd` module, run: 24 ./bp2build-progress report -m adbd 25 26 To generate a graph on the `adbd` module, run: 27 ./bp2build-progress graph -m adbd > graph.in && dot -Tpng -o graph.png 28 graph.in 29 30""" 31 32import argparse 33import collections 34import datetime 35import dependency_analysis 36import os.path 37import queue 38import subprocess 39import sys 40 41_ModuleInfo = collections.namedtuple("_ModuleInfo", [ 42 "name", 43 "kind", 44 "dirname", 45]) 46 47_ReportData = collections.namedtuple("_ReportData", [ 48 "input_module", 49 "all_unconverted_modules", 50 "blocked_modules", 51 "dirs_with_unconverted_modules", 52 "kind_of_unconverted_modules", 53 "converted", 54]) 55 56 57def combine_report_data(data): 58 ret = _ReportData( 59 input_module=set(), 60 all_unconverted_modules=collections.defaultdict(set), 61 blocked_modules=collections.defaultdict(set), 62 dirs_with_unconverted_modules=set(), 63 kind_of_unconverted_modules=set(), 64 converted=set(), 65 ) 66 for item in data: 67 ret.input_module.add(item.input_module) 68 for key, value in item.all_unconverted_modules.items(): 69 ret.all_unconverted_modules[key].update(value) 70 for key, value in item.blocked_modules.items(): 71 ret.blocked_modules[key].update(value) 72 ret.dirs_with_unconverted_modules.update(item.dirs_with_unconverted_modules) 73 ret.kind_of_unconverted_modules.update(item.kind_of_unconverted_modules) 74 if len(ret.converted) == 0: 75 ret.converted.update(item.converted) 76 return ret 77 78 79# Generate a dot file containing the transitive closure of the module. 80def generate_dot_file(modules, converted, module): 81 DOT_TEMPLATE = """ 82digraph mygraph {{ 83 node [shape=box]; 84 85 %s 86}} 87""" 88 89 make_node = lambda module, color: \ 90 ('"{name}" [label="{name}\\n{kind}" color=black, style=filled, ' 91 "fillcolor={color}]").format(name=module.name, kind=module.kind, color=color) 92 make_edge = lambda module, dep: \ 93 '"%s" -> "%s"' % (module.name, dep) 94 95 # Check that all modules in the argument are in the list of converted modules 96 all_converted = lambda modules: all(map(lambda m: m in converted, modules)) 97 98 dot_entries = [] 99 100 for module, deps in modules.items(): 101 if module.name in converted: 102 # Skip converted modules (nodes) 103 continue 104 elif module.name not in converted: 105 if all_converted(deps): 106 dot_entries.append(make_node(module, "yellow")) 107 else: 108 dot_entries.append(make_node(module, "tomato")) 109 110 # Print all edges for this module 111 for dep in list(deps): 112 # Skip converted deps (edges) 113 if dep not in converted: 114 dot_entries.append(make_edge(module, dep)) 115 116 print(DOT_TEMPLATE % "\n ".join(dot_entries)) 117 118 119# Generate a report for each module in the transitive closure, and the blockers for each module 120def generate_report_data(modules, converted, input_module): 121 # Map of [number of unconverted deps] to list of entries, 122 # with each entry being the string: "<module>: <comma separated list of unconverted modules>" 123 blocked_modules = collections.defaultdict(set) 124 125 # Map of unconverted modules to the modules they're blocking 126 # (i.e. reverse deps) 127 all_unconverted_modules = collections.defaultdict(set) 128 129 dirs_with_unconverted_modules = set() 130 kind_of_unconverted_modules = set() 131 132 for module, deps in sorted(modules.items()): 133 unconverted_deps = set(dep for dep in deps if dep not in converted) 134 for dep in unconverted_deps: 135 all_unconverted_modules[dep].add(module) 136 137 unconverted_count = len(unconverted_deps) 138 if module.name not in converted: 139 report_entry = "{name} [{kind}] [{dirname}]: {unconverted_deps}".format( 140 name=module.name, 141 kind=module.kind, 142 dirname=module.dirname, 143 unconverted_deps=", ".join(sorted(unconverted_deps))) 144 blocked_modules[unconverted_count].add(report_entry) 145 dirs_with_unconverted_modules.add(module.dirname) 146 kind_of_unconverted_modules.add(module.kind) 147 148 return _ReportData( 149 input_module=input_module, 150 all_unconverted_modules=all_unconverted_modules, 151 blocked_modules=blocked_modules, 152 dirs_with_unconverted_modules=dirs_with_unconverted_modules, 153 kind_of_unconverted_modules=kind_of_unconverted_modules, 154 converted=converted, 155 ) 156 157 158def generate_report(report_data): 159 report_lines = [] 160 input_modules = sorted(report_data.input_module) 161 162 report_lines.append("# bp2build progress report for: %s\n" % input_modules) 163 report_lines.append("Ignored module types: %s\n" % 164 sorted(dependency_analysis.IGNORED_KINDS)) 165 report_lines.append("# Transitive dependency closure:") 166 167 for count, modules in sorted(report_data.blocked_modules.items()): 168 report_lines.append("\n%d unconverted deps remaining:" % count) 169 for module_string in sorted(modules): 170 report_lines.append(" " + module_string) 171 172 report_lines.append("\n") 173 report_lines.append("# Unconverted deps of {}:\n".format(input_modules)) 174 for count, dep in sorted( 175 ((len(unconverted), dep) 176 for dep, unconverted in report_data.all_unconverted_modules.items()), 177 reverse=True): 178 report_lines.append("%s: blocking %d modules" % (dep, count)) 179 180 report_lines.append("\n") 181 report_lines.append("# Dirs with unconverted modules:\n\n{}".format("\n".join( 182 sorted(report_data.dirs_with_unconverted_modules)))) 183 184 report_lines.append("\n") 185 report_lines.append("# Kinds with unconverted modules:\n\n{}".format( 186 "\n".join(sorted(report_data.kind_of_unconverted_modules)))) 187 188 report_lines.append("\n") 189 report_lines.append("# Converted modules:\n\n%s" % 190 "\n".join(sorted(report_data.converted))) 191 192 report_lines.append("\n") 193 report_lines.append( 194 "Generated by: https://cs.android.com/android/platform/superproject/+/master:build/bazel/scripts/bp2build-progress/bp2build-progress.py" 195 ) 196 report_lines.append("Generated at: %s" % 197 datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S %z")) 198 print("\n".join(report_lines)) 199 200 201def adjacency_list_from_json(module_graph, ignore_by_name, top_level_module): 202 # The set of ignored modules. These modules (and their dependencies) are not shown 203 # in the graph or report. 204 ignored = set() 205 206 # A map of module name to _ModuleInfo 207 name_to_info = dict() 208 module_graph_map = dict() 209 q = queue.Queue() 210 211 # Do a single pass to find all top-level modules to be ignored 212 for module in module_graph: 213 name = module["Name"] 214 if dependency_analysis.is_windows_variation(module): 215 continue 216 if ignore_kind(module["Type"]) or name in ignore_by_name: 217 ignored.add(module["Name"]) 218 continue 219 name_to_info[name] = _ModuleInfo( 220 name=name, 221 kind=module["Type"], 222 dirname=os.path.dirname(module["Blueprint"])) 223 module_graph_map[module["Name"]] = module 224 if module["Name"] == top_level_module: 225 q.put(module["Name"]) 226 227 # An adjacency list for all modules in the transitive closure, excluding ignored modules. 228 module_adjacency_list = {} 229 visited = set() 230 # Create the adjacency list. 231 while not q.empty(): 232 module_name = q.get() 233 module = module_graph_map[module_name] 234 visited.add(module_name) 235 if module_name in ignored: 236 continue 237 if dependency_analysis.is_windows_variation(module): 238 # ignore the windows variations of modules 239 continue 240 241 module_info = name_to_info[module_name] 242 module_adjacency_list[module_info] = set() 243 for dep in module["Deps"]: 244 dep_name = dep["Name"] 245 if dep_name in ignored or dep_name == module_name: 246 continue 247 module_adjacency_list[module_info].add(dep_name) 248 if dep_name not in visited: 249 q.put(dep_name) 250 251 return module_adjacency_list 252 253 254def ignore_kind(kind): 255 return kind in dependency_analysis.IGNORED_KINDS or "defaults" in kind 256 257 258def bazel_target_to_dir(full_target): 259 dirname, _ = full_target.split(":") 260 return dirname[2:] 261 262 263def adjacency_list_from_queryview_xml(module_graph, ignore_by_name, 264 top_level_module): 265 # The set of ignored modules. These modules (and their dependencies) are 266 # not shown in the graph or report. 267 ignored = set() 268 269 # A map of module name to ModuleInfo 270 name_to_info = dict() 271 272 # queryview embeds variant in long name, keep a map of the name with vaiarnt 273 # to just name 274 name_with_variant_to_name = dict() 275 276 module_graph_map = dict() 277 q = queue.Queue() 278 279 for module in module_graph: 280 ignore = False 281 if module.tag != "rule": 282 continue 283 kind = module.attrib["class"] 284 name_with_variant = module.attrib["name"] 285 name = None 286 variant = "" 287 for attr in module: 288 attr_name = attr.attrib["name"] 289 if attr_name == "soong_module_name": 290 name = attr.attrib["value"] 291 elif attr_name == "soong_module_variant": 292 variant = attr.attrib["value"] 293 elif attr_name == "soong_module_type" and kind == "generic_soong_module": 294 kind = attr.attrib["value"] 295 # special handling for filegroup srcs, if a source has the same name as 296 # the module, we don't convert it 297 elif kind == "filegroup" and attr_name == "srcs": 298 for item in attr: 299 if item.attrib["value"] == name: 300 ignore = True 301 if name in ignore_by_name: 302 ignore = True 303 304 if ignore_kind(kind) or variant.startswith("windows") or ignore: 305 ignored.add(name_with_variant) 306 else: 307 if name == top_level_module: 308 q.put(name_with_variant) 309 name_with_variant_to_name.setdefault(name_with_variant, name) 310 name_to_info.setdefault( 311 name, 312 _ModuleInfo( 313 name=name, 314 kind=kind, 315 dirname=bazel_target_to_dir(name_with_variant), 316 )) 317 module_graph_map[name_with_variant] = module 318 319 # An adjacency list for all modules in the transitive closure, excluding ignored modules. 320 module_adjacency_list = dict() 321 visited = set() 322 while not q.empty(): 323 name_with_variant = q.get() 324 module = module_graph_map[name_with_variant] 325 if module.tag != "rule": 326 continue 327 visited.add(name_with_variant) 328 if name_with_variant in ignored: 329 continue 330 331 name = name_with_variant_to_name[name_with_variant] 332 module_info = name_to_info[name] 333 module_adjacency_list[module_info] = set() 334 for attr in module: 335 if attr.tag != "rule-input": 336 continue 337 dep_name_with_variant = attr.attrib["name"] 338 if dep_name_with_variant in ignored: 339 continue 340 dep_name = name_with_variant_to_name[dep_name_with_variant] 341 if name == dep_name: 342 continue 343 if dep_name_with_variant not in visited: 344 q.put(dep_name_with_variant) 345 module_adjacency_list[module_info].add(dep_name) 346 347 return module_adjacency_list 348 349 350def get_module_adjacency_list(top_level_module, use_queryview, ignore_by_name): 351 # The main module graph containing _all_ modules in the Soong build, 352 # and the list of converted modules. 353 try: 354 module_graph = dependency_analysis.get_queryview_module_info( 355 top_level_module 356 ) if use_queryview else dependency_analysis.get_json_module_info( 357 top_level_module) 358 converted = dependency_analysis.get_bp2build_converted_modules() 359 except subprocess.CalledProcessError as err: 360 print("Error running: '%s':", " ".join(err.cmd)) 361 print("Output:\n%s" % err.output.decode("utf-8")) 362 print("Error:\n%s" % err.stderr.decode("utf-8")) 363 sys.exit(-1) 364 365 module_adjacency_list = None 366 if use_queryview: 367 module_adjacency_list = adjacency_list_from_queryview_xml( 368 module_graph, ignore_by_name, top_level_module) 369 else: 370 module_adjacency_list = adjacency_list_from_json(module_graph, 371 ignore_by_name, 372 top_level_module) 373 374 return module_adjacency_list, converted 375 376 377def main(): 378 parser = argparse.ArgumentParser(description="") 379 parser.add_argument("mode", help="mode: graph or report") 380 parser.add_argument( 381 "--module", 382 "-m", 383 action="append", 384 help="name(s) of Soong module(s). Multiple modules only supported for report" 385 ) 386 parser.add_argument( 387 "--use_queryview", 388 type=bool, 389 default=False, 390 required=False, 391 help="whether to use queryview or module_info") 392 parser.add_argument( 393 "--ignore_by_name", 394 type=str, 395 default="", 396 required=False, 397 help="Comma-separated list. When building the tree of transitive dependencies, will not follow dependency edges pointing to module names listed by this flag." 398 ) 399 args = parser.parse_args() 400 401 if len(args.module) > 1 and args.mode != "report": 402 print("Can only support one module with mode {}", args.mode) 403 404 mode = args.mode 405 use_queryview = args.use_queryview 406 ignore_by_name = args.ignore_by_name 407 408 report_infos = [] 409 for top_level_module in args.module: 410 module_adjacency_list, converted = get_module_adjacency_list( 411 top_level_module, use_queryview, ignore_by_name) 412 413 if mode == "graph": 414 generate_dot_file(module_adjacency_list, converted, top_level_module) 415 elif mode == "report": 416 report_infos.append( 417 generate_report_data(module_adjacency_list, converted, 418 top_level_module)) 419 else: 420 raise RuntimeError("unknown mode: %s" % mode) 421 422 if mode == "report": 423 combined_data = combine_report_data(report_infos) 424 generate_report(combined_data) 425 426 427if __name__ == "__main__": 428 main() 429