• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#   http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""A json-module-graph postprocessing script to generate a bp2build progress tracker.
17
18Usage:
19  b run :bp2build_progress [report|graph] -m <module name>
20
21Example:
22
23  To generate a report on the `adbd` module, run:
24    b run //build/bazel/scripts/bp2build_progress:bp2build_progress \
25      -- report -m <module-name>
26
27  To generate a graph on the `adbd` module, run:
28    b run //build/bazel/scripts/bp2build_progress:bp2build_progress \
29      -- graph -m adbd > /tmp/graph.in && \
30      dot -Tpng -o /tmp/graph.png /tmp/graph.in
31"""
32
33import argparse
34import collections
35import dataclasses
36import datetime
37import os.path
38import subprocess
39import sys
40import xml
41from typing import Dict, List, Set, Optional
42
43import bp2build_pb2
44import dependency_analysis
45
46
47@dataclasses.dataclass(frozen=True, order=True)
48class ModuleInfo:
49  name: str
50  kind: str
51  dirname: str
52  created_by: Optional[str]
53  num_deps: int = 0
54  converted: bool = False
55
56  def __str__(self):
57    converted = " (converted)" if self.converted else ""
58    return f"{self.name} [{self.kind}] [{self.dirname}]{converted}"
59
60  def short_string(self, converted: Set[str]):
61    converted = " (c)" if self.is_converted(converted) else ""
62    return f"{self.name}{converted}"
63
64  def is_converted(self, converted: Set[str]):
65    return self.name in converted
66
67  def is_converted_or_skipped(self, converted: Set[str]):
68    if self.is_converted(converted):
69      return True
70    # these are implementation details of another module type that can never be
71    # created in a BUILD file
72    return ".go_android/soong" in self.kind and (
73        self.kind.endswith("__loadHookModule") or
74        self.kind.endswith("__topDownMutatorModule"))
75
76
77@dataclasses.dataclass(frozen=True, order=True)
78class InputModule:
79  module: ModuleInfo
80  num_deps: int = 0
81  num_unconverted_deps: int = 0
82
83  def __str__(self):
84    total = self.num_deps
85    converted = self.num_deps - self.num_unconverted_deps
86    percent = 1
87    if self.num_deps > 0:
88      percent = converted / self.num_deps * 100
89    return f"{self.module.name}: {percent:.1f}% ({converted}/{total}) converted"
90
91
92@dataclasses.dataclass(frozen=True)
93class ReportData:
94  input_modules: Set[InputModule]
95  total_deps: Set[ModuleInfo]
96  unconverted_deps: Set[str]
97  all_unconverted_modules: Dict[str, Set[ModuleInfo]]
98  blocked_modules: Dict[ModuleInfo, Set[str]]
99  dirs_with_unconverted_modules: Set[str]
100  kind_of_unconverted_modules: Set[str]
101  converted: Set[str]
102  show_converted: bool
103
104
105# Generate a dot file containing the transitive closure of the module.
106def generate_dot_file(modules: Dict[ModuleInfo, Set[ModuleInfo]],
107                      converted: Set[str], show_converted: bool):
108  # Check that all modules in the argument are in the list of converted modules
109  all_converted = lambda modules: all(
110      m.is_converted(converted) for m in modules)
111
112  dot_entries = []
113
114  for module, deps in sorted(modules.items()):
115
116    if module.is_converted(converted):
117      if show_converted:
118        color = "dodgerblue"
119      else:
120        continue
121    elif all_converted(deps):
122      color = "yellow"
123    else:
124      color = "tomato"
125
126    dot_entries.append(
127        f'"{module.name}" [label="{module.name}\\n{module.kind}" color=black, style=filled, '
128        f"fillcolor={color}]")
129    dot_entries.extend(
130        f'"{module.name}" -> "{dep.name}"' for dep in sorted(deps)
131        if show_converted or not dep.is_converted(converted))
132
133  return """
134digraph mygraph {{
135  node [shape=box];
136
137  %s
138}}
139""" % "\n  ".join(dot_entries)
140
141
142# Generate a report for each module in the transitive closure, and the blockers for each module
143def generate_report_data(modules: Dict[ModuleInfo, Set[ModuleInfo]],
144                         converted: Set[str],
145                         input_modules_names: Set[str],
146                         show_converted: bool = False) -> ReportData:
147  # Map of [number of unconverted deps] to list of entries,
148  # with each entry being the string: "<module>: <comma separated list of unconverted modules>"
149  blocked_modules = collections.defaultdict(set)
150
151  # Map of unconverted modules to the modules they're blocking
152  # (i.e. reverse deps)
153  all_unconverted_modules = collections.defaultdict(set)
154
155  dirs_with_unconverted_modules = set()
156  kind_of_unconverted_modules = set()
157
158  input_all_deps = set()
159  input_unconverted_deps = set()
160  input_modules = set()
161
162  for module, deps in sorted(modules.items()):
163    unconverted_deps = set(
164        dep.name for dep in deps if not dep.is_converted_or_skipped(converted))
165
166    # replace deps count with transitive deps rather than direct deps count
167    module = ModuleInfo(
168        module.name,
169        module.kind,
170        module.dirname,
171        module.created_by,
172        len(deps),
173        module.is_converted(converted),
174    )
175
176    for dep in unconverted_deps:
177      all_unconverted_modules[dep].add(module)
178
179    if not module.is_converted_or_skipped(converted) or (
180        show_converted and not module.is_converted_or_skipped(set())):
181      if show_converted:
182        full_deps = set(f"{dep.short_string(converted)}" for dep in deps)
183        blocked_modules[module].update(full_deps)
184      else:
185        blocked_modules[module].update(unconverted_deps)
186
187    if not module.is_converted_or_skipped(converted):
188      dirs_with_unconverted_modules.add(module.dirname)
189      kind_of_unconverted_modules.add(module.kind)
190
191    if module.name in input_modules_names:
192      input_modules.add(InputModule(module, len(deps), len(unconverted_deps)))
193      input_all_deps.update(deps)
194      input_unconverted_deps.update(unconverted_deps)
195
196  return ReportData(
197      input_modules=input_modules,
198      total_deps=input_all_deps,
199      unconverted_deps=input_unconverted_deps,
200      all_unconverted_modules=all_unconverted_modules,
201      blocked_modules=blocked_modules,
202      dirs_with_unconverted_modules=dirs_with_unconverted_modules,
203      kind_of_unconverted_modules=kind_of_unconverted_modules,
204      converted=converted,
205      show_converted=show_converted,
206  )
207
208
209def generate_proto(report_data, file_name):
210  message = bp2build_pb2.Bp2buildConversionProgress(
211      root_modules=[m.module.name for m in report_data.input_modules],
212      num_deps=len(report_data.total_deps),
213  )
214  for module, unconverted_deps in report_data.blocked_modules.items():
215    message.unconverted.add(
216        name=module.name,
217        directory=module.dirname,
218        type=module.kind,
219        unconverted_deps=unconverted_deps,
220        num_deps=module.num_deps,
221    )
222
223  with open(file_name, "wb") as f:
224    f.write(message.SerializeToString())
225
226
227def generate_report(report_data):
228  report_lines = []
229  input_module_str = ", ".join(
230      str(i) for i in sorted(report_data.input_modules))
231
232  report_lines.append("# bp2build progress report for: %s\n" % input_module_str)
233
234  if report_data.show_converted:
235    report_lines.append(
236        "# progress report includes data both for converted and unconverted modules"
237    )
238
239  total = len(report_data.total_deps)
240  unconverted = len(report_data.unconverted_deps)
241  converted = total - unconverted
242  percent = converted / total * 100
243  report_lines.append(f"Percent converted: {percent:.2f} ({converted}/{total})")
244  report_lines.append(f"Total unique unconverted dependencies: {unconverted}")
245
246  report_lines.append("Ignored module types: %s\n" %
247                      sorted(dependency_analysis.IGNORED_KINDS))
248  report_lines.append("# Transitive dependency closure:")
249
250  current_count = -1
251  for module, unconverted_deps in sorted(
252      report_data.blocked_modules.items(), key=lambda x: len(x[1])):
253    count = len(unconverted_deps)
254    if current_count != count:
255      report_lines.append(f"\n{count} unconverted deps remaining:")
256      current_count = count
257    report_lines.append("{module}: {deps}".format(
258        module=module, deps=", ".join(sorted(unconverted_deps))))
259
260  report_lines.append("\n")
261  report_lines.append("# Unconverted deps of {}:\n".format(input_module_str))
262  for count, dep in sorted(
263      ((len(unconverted), dep)
264       for dep, unconverted in report_data.all_unconverted_modules.items()),
265      reverse=True):
266    report_lines.append("%s: blocking %d modules" % (dep, count))
267
268  report_lines.append("\n")
269  report_lines.append("# Dirs with unconverted modules:\n\n{}".format("\n".join(
270      sorted(report_data.dirs_with_unconverted_modules))))
271
272  report_lines.append("\n")
273  report_lines.append("# Kinds with unconverted modules:\n\n{}".format(
274      "\n".join(sorted(report_data.kind_of_unconverted_modules))))
275
276  report_lines.append("\n")
277  report_lines.append("# Converted modules:\n\n%s" %
278                      "\n".join(sorted(report_data.converted)))
279
280  report_lines.append("\n")
281  report_lines.append(
282      "Generated by: https://cs.android.com/android/platform/superproject/+/master:build/bazel/scripts/bp2build_progress/bp2build_progress.py"
283  )
284  report_lines.append("Generated at: %s" %
285                      datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S %z"))
286
287  return "\n".join(report_lines)
288
289
290def adjacency_list_from_json(
291    module_graph: ...,
292    ignore_by_name: List[str],
293    ignore_java_auto_deps: bool,
294    top_level_modules: List[str],
295    collect_transitive_dependencies: bool = True,
296) -> Dict[ModuleInfo, Set[ModuleInfo]]:
297  def filter_by_name(json):
298    return json["Name"] in top_level_modules
299
300  module_adjacency_list = collections.defaultdict(set)
301  name_to_info = {}
302
303  def collect_dependencies(module, deps_names):
304    module_info = None
305    name = module["Name"]
306    name_to_info.setdefault(
307        name,
308        ModuleInfo(
309            name=name,
310            created_by=module["CreatedBy"],
311            kind=module["Type"],
312            dirname=os.path.dirname(module["Blueprint"]),
313            num_deps=len(deps_names),
314        ))
315
316    module_info = name_to_info[name]
317
318    # ensure module_info added to adjacency list even with no deps
319    module_adjacency_list[module_info].update(set())
320    for dep in deps_names:
321      # this may occur if there is a cycle between a module and created_by
322      # module
323      if not dep in name_to_info:
324        continue
325      dep_module_info = name_to_info[dep]
326      module_adjacency_list[module_info].add(dep_module_info)
327      if collect_transitive_dependencies:
328        module_adjacency_list[module_info].update(
329            module_adjacency_list.get(dep_module_info, set()))
330
331  dependency_analysis.visit_json_module_graph_post_order(
332      module_graph, ignore_by_name, ignore_java_auto_deps, filter_by_name, collect_dependencies)
333
334  return module_adjacency_list
335
336
337def adjacency_list_from_queryview_xml(
338    module_graph: xml.etree.ElementTree,
339    ignore_by_name: List[str],
340    top_level_modules: List[str],
341    collect_transitive_dependencies: bool = True
342) -> Dict[ModuleInfo, Set[ModuleInfo]]:
343
344  def filter_by_name(module):
345    return module.name in top_level_modules
346
347  module_adjacency_list = collections.defaultdict(set)
348  name_to_info = {}
349
350  def collect_dependencies(module, deps_names):
351    module_info = None
352    name_to_info.setdefault(
353        module.name,
354        ModuleInfo(
355            name=module.name,
356            kind=module.kind,
357            dirname=module.dirname,
358            # required so that it cannot be forgotten when updating num_deps
359            created_by=None,
360            num_deps=len(deps_names),
361        ))
362    module_info = name_to_info[module.name]
363
364    # ensure module_info added to adjacency list even with no deps
365    module_adjacency_list[module_info].update(set())
366    for dep in deps_names:
367      dep_module_info = name_to_info[dep]
368      module_adjacency_list[module_info].add(dep_module_info)
369      if collect_transitive_dependencies:
370        module_adjacency_list[module_info].update(
371            module_adjacency_list.get(dep_module_info, set()))
372
373  dependency_analysis.visit_queryview_xml_module_graph_post_order(
374      module_graph, ignore_by_name, filter_by_name, collect_dependencies)
375
376  return module_adjacency_list
377
378
379def get_module_adjacency_list(
380    top_level_modules: List[str],
381    use_queryview: bool,
382    ignore_by_name: List[str],
383    ignore_java_auto_deps: bool = False,
384    collect_transitive_dependencies: bool = True,
385    banchan_mode: bool = False) -> Dict[ModuleInfo, Set[ModuleInfo]]:
386  # The main module graph containing _all_ modules in the Soong build,
387  # and the list of converted modules.
388  try:
389    if use_queryview:
390      module_graph = dependency_analysis.get_queryview_module_info(
391          top_level_modules, banchan_mode)
392      module_adjacency_list = adjacency_list_from_queryview_xml(
393          module_graph, ignore_by_name, top_level_modules,
394          collect_transitive_dependencies)
395    else:
396      module_graph = dependency_analysis.get_json_module_info(banchan_mode)
397      module_adjacency_list = adjacency_list_from_json(
398          module_graph,
399          ignore_by_name,
400          ignore_java_auto_deps,
401          top_level_modules,
402          collect_transitive_dependencies,
403      )
404  except subprocess.CalledProcessError as err:
405    sys.exit(f"""Error running: '{' '.join(err.cmd)}':"
406Stdout:
407{err.stdout.decode('utf-8') if err.stdout else ''}
408Stderr:
409{err.stderr.decode('utf-8') if err.stderr else ''}""")
410
411  return module_adjacency_list
412
413
414def add_created_by_to_converted(
415    converted: Set[str],
416    module_adjacency_list: Dict[ModuleInfo, Set[ModuleInfo]]) -> Set[str]:
417  modules_by_name = {m.name: m for m in module_adjacency_list.keys()}
418
419  converted_modules = set()
420  converted_modules.update(converted)
421
422  def _update_converted(module_name):
423    if module_name in converted_modules:
424      return True
425    if module_name not in modules_by_name:
426      return False
427    module = modules_by_name[module_name]
428    if module.created_by and _update_converted(module.created_by):
429      converted_modules.add(module_name)
430      return True
431    return False
432
433  for module in modules_by_name.keys():
434    _update_converted(module)
435
436  return converted_modules
437
438
439def main():
440  parser = argparse.ArgumentParser()
441  parser.add_argument("mode", help="mode: graph or report")
442  parser.add_argument(
443      "--module",
444      "-m",
445      action="append",
446      required=True,
447      help="name(s) of Soong module(s). Multiple modules only supported for report"
448  )
449  parser.add_argument(
450      "--use-queryview",
451      action="store_true",
452      help="whether to use queryview or module_info")
453  parser.add_argument(
454      "--ignore-by-name",
455      default="",
456      help=(
457          "Comma-separated list. When building the tree of transitive"
458          " dependencies, will not follow dependency edges pointing to module"
459          " names listed by this flag."
460      ),
461  )
462  parser.add_argument(
463      "--ignore-java-auto-deps",
464      action="store_true",
465      help="whether to ignore automatically added java deps",
466  )
467  parser.add_argument(
468      "--banchan",
469      action="store_true",
470      help="whether to run Soong in a banchan configuration rather than lunch",
471  )
472  parser.add_argument(
473      "--proto-file",
474      help="Path to write proto output",
475  )
476  parser.add_argument(
477      "--out-file",
478      "-o",
479      type=argparse.FileType("w"),
480      default="-",
481      help="Path to write output, if omitted, writes to stdout",
482  )
483  parser.add_argument(
484      "--show-converted",
485      "-s",
486      action="store_true",
487      help="Show bp2build-converted modules in addition to the unconverted dependencies to see full dependencies post-migration. By default converted dependencies are not shown",
488  )
489  args = parser.parse_args()
490
491  if len(args.module) > 1 and args.mode == "graph":
492    sys.exit(f"Can only support one module with mode {args.mode}")
493  if args.proto_file and args.mode == "graph":
494    sys.exit(f"Proto file only supported for report mode, not {args.mode}")
495
496  mode = args.mode
497  use_queryview = args.use_queryview
498  ignore_by_name = args.ignore_by_name.split(",")
499  banchan_mode = args.banchan
500  modules = set(args.module)
501
502  converted = dependency_analysis.get_bp2build_converted_modules()
503
504  module_adjacency_list = get_module_adjacency_list(
505      modules,
506      use_queryview,
507      ignore_by_name,
508      collect_transitive_dependencies=mode != "graph",
509      banchan_mode=banchan_mode)
510
511  converted = add_created_by_to_converted(converted, module_adjacency_list)
512
513  output_file = args.out_file
514  if mode == "graph":
515    dot_file = generate_dot_file(module_adjacency_list, converted,
516                                 args.show_converted)
517    output_file.write(dot_file)
518  elif mode == "report":
519    report_data = generate_report_data(module_adjacency_list, converted,
520                                       modules, args.show_converted)
521    report = generate_report(report_data)
522    output_file.write(report)
523    if args.proto_file:
524      generate_proto(report_data, args.proto_file)
525  else:
526    raise RuntimeError("unknown mode: %s" % mode)
527
528
529if __name__ == "__main__":
530  main()
531