• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#   http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""A json-module-graph postprocessing script to generate a bp2build progress tracker.
17
18Usage:
19  ./bp2build-progress.py [report|graph] -m <module name>
20
21Example:
22
23  To generate a report on the `adbd` module, run:
24    ./bp2build-progress report -m adbd
25
26  To generate a graph on the `adbd` module, run:
27    ./bp2build-progress graph -m adbd > graph.in && dot -Tpng -o graph.png
28    graph.in
29
30"""
31
32import argparse
33import collections
34import datetime
35import dependency_analysis
36import os.path
37import queue
38import subprocess
39import sys
40
41_ModuleInfo = collections.namedtuple("_ModuleInfo", [
42    "name",
43    "kind",
44    "dirname",
45])
46
47_ReportData = collections.namedtuple("_ReportData", [
48    "input_module",
49    "all_unconverted_modules",
50    "blocked_modules",
51    "dirs_with_unconverted_modules",
52    "kind_of_unconverted_modules",
53    "converted",
54])
55
56
57def combine_report_data(data):
58  ret = _ReportData(
59      input_module=set(),
60      all_unconverted_modules=collections.defaultdict(set),
61      blocked_modules=collections.defaultdict(set),
62      dirs_with_unconverted_modules=set(),
63      kind_of_unconverted_modules=set(),
64      converted=set(),
65  )
66  for item in data:
67    ret.input_module.add(item.input_module)
68    for key, value in item.all_unconverted_modules.items():
69      ret.all_unconverted_modules[key].update(value)
70    for key, value in item.blocked_modules.items():
71      ret.blocked_modules[key].update(value)
72    ret.dirs_with_unconverted_modules.update(item.dirs_with_unconverted_modules)
73    ret.kind_of_unconverted_modules.update(item.kind_of_unconverted_modules)
74    if len(ret.converted) == 0:
75      ret.converted.update(item.converted)
76  return ret
77
78
79# Generate a dot file containing the transitive closure of the module.
80def generate_dot_file(modules, converted, module):
81  DOT_TEMPLATE = """
82digraph mygraph {{
83  node [shape=box];
84
85  %s
86}}
87"""
88
89  make_node = lambda module, color: \
90      ('"{name}" [label="{name}\\n{kind}" color=black, style=filled, '
91       "fillcolor={color}]").format(name=module.name, kind=module.kind, color=color)
92  make_edge = lambda module, dep: \
93      '"%s" -> "%s"' % (module.name, dep)
94
95  # Check that all modules in the argument are in the list of converted modules
96  all_converted = lambda modules: all(map(lambda m: m in converted, modules))
97
98  dot_entries = []
99
100  for module, deps in modules.items():
101    if module.name in converted:
102      # Skip converted modules (nodes)
103      continue
104    elif module.name not in converted:
105      if all_converted(deps):
106        dot_entries.append(make_node(module, "yellow"))
107      else:
108        dot_entries.append(make_node(module, "tomato"))
109
110    # Print all edges for this module
111    for dep in list(deps):
112      # Skip converted deps (edges)
113      if dep not in converted:
114        dot_entries.append(make_edge(module, dep))
115
116  print(DOT_TEMPLATE % "\n  ".join(dot_entries))
117
118
119# Generate a report for each module in the transitive closure, and the blockers for each module
120def generate_report_data(modules, converted, input_module):
121  # Map of [number of unconverted deps] to list of entries,
122  # with each entry being the string: "<module>: <comma separated list of unconverted modules>"
123  blocked_modules = collections.defaultdict(set)
124
125  # Map of unconverted modules to the modules they're blocking
126  # (i.e. reverse deps)
127  all_unconverted_modules = collections.defaultdict(set)
128
129  dirs_with_unconverted_modules = set()
130  kind_of_unconverted_modules = set()
131
132  for module, deps in sorted(modules.items()):
133    unconverted_deps = set(dep for dep in deps if dep not in converted)
134    for dep in unconverted_deps:
135      all_unconverted_modules[dep].add(module)
136
137    unconverted_count = len(unconverted_deps)
138    if module.name not in converted:
139      report_entry = "{name} [{kind}] [{dirname}]: {unconverted_deps}".format(
140          name=module.name,
141          kind=module.kind,
142          dirname=module.dirname,
143          unconverted_deps=", ".join(sorted(unconverted_deps)))
144      blocked_modules[unconverted_count].add(report_entry)
145      dirs_with_unconverted_modules.add(module.dirname)
146      kind_of_unconverted_modules.add(module.kind)
147
148  return _ReportData(
149      input_module=input_module,
150      all_unconverted_modules=all_unconverted_modules,
151      blocked_modules=blocked_modules,
152      dirs_with_unconverted_modules=dirs_with_unconverted_modules,
153      kind_of_unconverted_modules=kind_of_unconverted_modules,
154      converted=converted,
155  )
156
157
158def generate_report(report_data):
159  report_lines = []
160  input_modules = sorted(report_data.input_module)
161
162  report_lines.append("# bp2build progress report for: %s\n" % input_modules)
163  report_lines.append("Ignored module types: %s\n" %
164                      sorted(dependency_analysis.IGNORED_KINDS))
165  report_lines.append("# Transitive dependency closure:")
166
167  for count, modules in sorted(report_data.blocked_modules.items()):
168    report_lines.append("\n%d unconverted deps remaining:" % count)
169    for module_string in sorted(modules):
170      report_lines.append("  " + module_string)
171
172  report_lines.append("\n")
173  report_lines.append("# Unconverted deps of {}:\n".format(input_modules))
174  for count, dep in sorted(
175      ((len(unconverted), dep)
176       for dep, unconverted in report_data.all_unconverted_modules.items()),
177      reverse=True):
178    report_lines.append("%s: blocking %d modules" % (dep, count))
179
180  report_lines.append("\n")
181  report_lines.append("# Dirs with unconverted modules:\n\n{}".format("\n".join(
182      sorted(report_data.dirs_with_unconverted_modules))))
183
184  report_lines.append("\n")
185  report_lines.append("# Kinds with unconverted modules:\n\n{}".format(
186      "\n".join(sorted(report_data.kind_of_unconverted_modules))))
187
188  report_lines.append("\n")
189  report_lines.append("# Converted modules:\n\n%s" %
190                      "\n".join(sorted(report_data.converted)))
191
192  report_lines.append("\n")
193  report_lines.append(
194      "Generated by: https://cs.android.com/android/platform/superproject/+/master:build/bazel/scripts/bp2build-progress/bp2build-progress.py"
195  )
196  report_lines.append("Generated at: %s" %
197                      datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S %z"))
198  print("\n".join(report_lines))
199
200
201def adjacency_list_from_json(module_graph, ignore_by_name, top_level_module):
202  # The set of ignored modules. These modules (and their dependencies) are not shown
203  # in the graph or report.
204  ignored = set()
205
206  # A map of module name to _ModuleInfo
207  name_to_info = dict()
208  module_graph_map = dict()
209  q = queue.Queue()
210
211  # Do a single pass to find all top-level modules to be ignored
212  for module in module_graph:
213    name = module["Name"]
214    if dependency_analysis.is_windows_variation(module):
215      continue
216    if ignore_kind(module["Type"]) or name in ignore_by_name:
217      ignored.add(module["Name"])
218      continue
219    name_to_info[name] = _ModuleInfo(
220        name=name,
221        kind=module["Type"],
222        dirname=os.path.dirname(module["Blueprint"]))
223    module_graph_map[module["Name"]] = module
224    if module["Name"] == top_level_module:
225      q.put(module["Name"])
226
227  # An adjacency list for all modules in the transitive closure, excluding ignored modules.
228  module_adjacency_list = {}
229  visited = set()
230  # Create the adjacency list.
231  while not q.empty():
232    module_name = q.get()
233    module = module_graph_map[module_name]
234    visited.add(module_name)
235    if module_name in ignored:
236      continue
237    if dependency_analysis.is_windows_variation(module):
238      # ignore the windows variations of modules
239      continue
240
241    module_info = name_to_info[module_name]
242    module_adjacency_list[module_info] = set()
243    for dep in module["Deps"]:
244      dep_name = dep["Name"]
245      if dep_name in ignored or dep_name == module_name:
246        continue
247      module_adjacency_list[module_info].add(dep_name)
248      if dep_name not in visited:
249        q.put(dep_name)
250
251  return module_adjacency_list
252
253
254def ignore_kind(kind):
255  return kind in dependency_analysis.IGNORED_KINDS or "defaults" in kind
256
257
258def bazel_target_to_dir(full_target):
259  dirname, _ = full_target.split(":")
260  return dirname[2:]
261
262
263def adjacency_list_from_queryview_xml(module_graph, ignore_by_name,
264                                      top_level_module):
265  # The set of ignored modules. These modules (and their dependencies) are
266  # not shown in the graph or report.
267  ignored = set()
268
269  # A map of module name to ModuleInfo
270  name_to_info = dict()
271
272  # queryview embeds variant in long name, keep a map of the name with vaiarnt
273  # to just name
274  name_with_variant_to_name = dict()
275
276  module_graph_map = dict()
277  q = queue.Queue()
278
279  for module in module_graph:
280    ignore = False
281    if module.tag != "rule":
282      continue
283    kind = module.attrib["class"]
284    name_with_variant = module.attrib["name"]
285    name = None
286    variant = ""
287    for attr in module:
288      attr_name = attr.attrib["name"]
289      if attr_name == "soong_module_name":
290        name = attr.attrib["value"]
291      elif attr_name == "soong_module_variant":
292        variant = attr.attrib["value"]
293      elif attr_name == "soong_module_type" and kind == "generic_soong_module":
294        kind = attr.attrib["value"]
295      # special handling for filegroup srcs, if a source has the same name as
296      # the module, we don't convert it
297      elif kind == "filegroup" and attr_name == "srcs":
298        for item in attr:
299          if item.attrib["value"] == name:
300            ignore = True
301    if name in ignore_by_name:
302      ignore = True
303
304    if ignore_kind(kind) or variant.startswith("windows") or ignore:
305      ignored.add(name_with_variant)
306    else:
307      if name == top_level_module:
308        q.put(name_with_variant)
309      name_with_variant_to_name.setdefault(name_with_variant, name)
310      name_to_info.setdefault(
311          name,
312          _ModuleInfo(
313              name=name,
314              kind=kind,
315              dirname=bazel_target_to_dir(name_with_variant),
316          ))
317      module_graph_map[name_with_variant] = module
318
319  # An adjacency list for all modules in the transitive closure, excluding ignored modules.
320  module_adjacency_list = dict()
321  visited = set()
322  while not q.empty():
323    name_with_variant = q.get()
324    module = module_graph_map[name_with_variant]
325    if module.tag != "rule":
326      continue
327    visited.add(name_with_variant)
328    if name_with_variant in ignored:
329      continue
330
331    name = name_with_variant_to_name[name_with_variant]
332    module_info = name_to_info[name]
333    module_adjacency_list[module_info] = set()
334    for attr in module:
335      if attr.tag != "rule-input":
336        continue
337      dep_name_with_variant = attr.attrib["name"]
338      if dep_name_with_variant in ignored:
339        continue
340      dep_name = name_with_variant_to_name[dep_name_with_variant]
341      if name == dep_name:
342        continue
343      if dep_name_with_variant not in visited:
344        q.put(dep_name_with_variant)
345      module_adjacency_list[module_info].add(dep_name)
346
347  return module_adjacency_list
348
349
350def get_module_adjacency_list(top_level_module, use_queryview, ignore_by_name):
351  # The main module graph containing _all_ modules in the Soong build,
352  # and the list of converted modules.
353  try:
354    module_graph = dependency_analysis.get_queryview_module_info(
355        top_level_module
356    ) if use_queryview else dependency_analysis.get_json_module_info(
357        top_level_module)
358    converted = dependency_analysis.get_bp2build_converted_modules()
359  except subprocess.CalledProcessError as err:
360    print("Error running: '%s':", " ".join(err.cmd))
361    print("Output:\n%s" % err.output.decode("utf-8"))
362    print("Error:\n%s" % err.stderr.decode("utf-8"))
363    sys.exit(-1)
364
365  module_adjacency_list = None
366  if use_queryview:
367    module_adjacency_list = adjacency_list_from_queryview_xml(
368        module_graph, ignore_by_name, top_level_module)
369  else:
370    module_adjacency_list = adjacency_list_from_json(module_graph,
371                                                     ignore_by_name,
372                                                     top_level_module)
373
374  return module_adjacency_list, converted
375
376
377def main():
378  parser = argparse.ArgumentParser(description="")
379  parser.add_argument("mode", help="mode: graph or report")
380  parser.add_argument(
381      "--module",
382      "-m",
383      action="append",
384      help="name(s) of Soong module(s). Multiple modules only supported for report"
385  )
386  parser.add_argument(
387      "--use_queryview",
388      type=bool,
389      default=False,
390      required=False,
391      help="whether to use queryview or module_info")
392  parser.add_argument(
393      "--ignore_by_name",
394      type=str,
395      default="",
396      required=False,
397      help="Comma-separated list. When building the tree of transitive dependencies, will not follow dependency edges pointing to module names listed by this flag."
398  )
399  args = parser.parse_args()
400
401  if len(args.module) > 1 and args.mode != "report":
402    print("Can only support one module with mode {}", args.mode)
403
404  mode = args.mode
405  use_queryview = args.use_queryview
406  ignore_by_name = args.ignore_by_name
407
408  report_infos = []
409  for top_level_module in args.module:
410    module_adjacency_list, converted = get_module_adjacency_list(
411        top_level_module, use_queryview, ignore_by_name)
412
413    if mode == "graph":
414      generate_dot_file(module_adjacency_list, converted, top_level_module)
415    elif mode == "report":
416      report_infos.append(
417          generate_report_data(module_adjacency_list, converted,
418                               top_level_module))
419    else:
420      raise RuntimeError("unknown mode: %s" % mode)
421
422  if mode == "report":
423    combined_data = combine_report_data(report_infos)
424    generate_report(combined_data)
425
426
427if __name__ == "__main__":
428  main()
429