• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#! /usr/bin/python3 -B
2# -*- coding: utf-8 -*-
3#
4# Copyright (C) 2016 and later: Unicode, Inc. and others.
5# License & terms of use: http://www.unicode.org/copyright.html
6# Copyright (C) 2011-2015, International Business Machines
7# Corporation and others. All Rights Reserved.
8#
9# file name: depstest.py
10#
11# created on: 2011may24
12
13"""ICU dependency tester.
14
15This probably works only on Linux.
16
17The exit code is 0 if everything is fine, 1 for errors, 2 for only warnings.
18
19Sample invocation with an in-source build:
20  ~/icu/icu4c/source/test/depstest$ ./depstest.py ../../
21
22Sample invocation with an out-of-source build:
23  ~/icu/icu4c/source/test/depstest$ ./depstest.py ~/build/
24"""
25
26from __future__ import print_function
27
28__author__ = "Markus W. Scherer"
29
30import glob
31import os.path
32import subprocess
33import sys
34
35import dependencies
36
37_ignored_symbols = set()
38_obj_files = {}
39_symbols_to_files = {}
40_return_value = 0
41
42# Classes with vtables (and thus virtual methods).
43_virtual_classes = set()
44# Classes with weakly defined destructors.
45# nm shows a symbol class of "W" rather than "T".
46_weak_destructors = set()
47
48def iteritems(items):
49  """Python 2/3-compatible iteritems"""
50  try:
51    for v in items.iteritems():
52      yield v
53  except AttributeError:
54    for v in items.items():
55      yield v
56
57def _ReadObjFile(root_path, library_name, obj_name):
58  global _ignored_symbols, _obj_files, _symbols_to_files
59  global _virtual_classes, _weak_destructors
60  lib_obj_name = library_name + "/" + obj_name
61  if lib_obj_name in _obj_files:
62    print("Warning: duplicate .o file " + lib_obj_name)
63    _return_value = 2
64    return
65
66  path = os.path.join(root_path, library_name, obj_name)
67  nm_result = subprocess.Popen(["nm", "--demangle", "--format=sysv",
68                                "--extern-only", "--no-sort", path],
69                               stdout=subprocess.PIPE).communicate()[0]
70  obj_imports = set()
71  obj_exports = set()
72  for line in nm_result.splitlines():
73    fields = line.decode().split("|")
74    if len(fields) == 1: continue
75    name = fields[0].strip()
76    # Ignore symbols like '__cxa_pure_virtual',
77    # 'vtable for __cxxabiv1::__si_class_type_info' or
78    # 'DW.ref.__gxx_personality_v0'.
79    # '__dso_handle' belongs to __cxa_atexit().
80    if (name.startswith("__cxa") or "__cxxabi" in name or "__gxx" in name or
81        name == "__dso_handle"):
82      _ignored_symbols.add(name)
83      continue
84    type = fields[2].strip()
85    if type == "U":
86      obj_imports.add(name)
87    else:
88      obj_exports.add(name)
89      _symbols_to_files[name] = lib_obj_name
90      # Is this a vtable? E.g., "vtable for icu_49::ByteSink".
91      if name.startswith("vtable for icu"):
92        _virtual_classes.add(name[name.index("::") + 2:])
93      # Is this a destructor? E.g., "icu_49::ByteSink::~ByteSink()".
94      index = name.find("::~")
95      if index >= 0 and type == "W":
96        _weak_destructors.add(name[index + 3:name.index("(", index)])
97  _obj_files[lib_obj_name] = {"imports": obj_imports, "exports": obj_exports}
98
99def _ReadLibrary(root_path, library_name):
100  obj_paths = glob.glob(os.path.join(root_path, library_name, "*.o"))
101  for path in obj_paths:
102    _ReadObjFile(root_path, library_name, os.path.basename(path))
103
104# Dependencies that would otherwise be errors, but that are to be allowed
105# in a limited (not transitive) context.  List of (file_name, symbol)
106# TODO: Move this data to dependencies.txt?
107allowed_errors = (
108  ("common/umutex.o", "std::__throw_system_error(int)"),
109  ("common/umutex.o", "std::uncaught_exception()"),
110  ("common/umutex.o", "std::__once_callable"),
111  ("common/umutex.o", "std::__once_call"),
112  ("common/umutex.o", "__once_proxy"),
113  ("common/umutex.o", "__tls_get_addr"),
114  ("common/unifiedcache.o", "std::__throw_system_error(int)"),
115  # Some of the MessageFormat 2 modules reference exception-related symbols
116  # in instantiations of the `std::get()` method that gets an alternative
117  # from a `std::variant`.
118  # These instantiations of `std::get()` are only called by compiler-generated
119  # code (the implementations of built-in `swap()` methods for types
120  # that include a `std::variant`; and `std::__detail::__variant::__gen_vtable_impl()`,
121  # which constructs vtables. The MessageFormat 2 code itself only calls
122  # `std::get_if()`, which is exception-free; never `std::get()`.
123  ("i18n/messageformat2_data_model.o", "typeinfo for std::exception"),
124  ("i18n/messageformat2_data_model.o", "vtable for std::exception"),
125  ("i18n/messageformat2_data_model.o", "std::exception::~exception()"),
126  ("i18n/messageformat2_formattable.o", "typeinfo for std::exception"),
127  ("i18n/messageformat2_formattable.o", "vtable for std::exception"),
128  ("i18n/messageformat2_formattable.o", "std::exception::~exception()"),
129  ("i18n/messageformat2_function_registry.o", "typeinfo for std::exception"),
130  ("i18n/messageformat2_function_registry.o", "vtable for std::exception"),
131  ("i18n/messageformat2_function_registry.o", "std::exception::~exception()")
132)
133
134def _Resolve(name, parents):
135  global _ignored_symbols, _obj_files, _symbols_to_files, _return_value
136  item = dependencies.items[name]
137  item_type = item["type"]
138  if name in parents:
139    sys.exit("Error: %s %s has a circular dependency on itself: %s" %
140             (item_type, name, parents))
141  # Check if already cached.
142  exports = item.get("exports")
143  if exports != None: return item
144  # Calculate recursively.
145  parents.append(name)
146  imports = set()
147  exports = set()
148  system_symbols = item.get("system_symbols")
149  if system_symbols == None: system_symbols = item["system_symbols"] = set()
150  files = item.get("files")
151  if files:
152    for file_name in files:
153      obj_file = _obj_files[file_name]
154      imports |= obj_file["imports"]
155      exports |= obj_file["exports"]
156  imports -= exports | _ignored_symbols
157  deps = item.get("deps")
158  if deps:
159    for dep in deps:
160      dep_item = _Resolve(dep, parents)
161      # Detect whether this item needs to depend on dep,
162      # except when this item has no files, that is, when it is just
163      # a deliberate umbrella group or library.
164      dep_exports = dep_item["exports"]
165      dep_system_symbols = dep_item["system_symbols"]
166      if files and imports.isdisjoint(dep_exports) and imports.isdisjoint(dep_system_symbols):
167        print("Info:  %s %s  does not need to depend on  %s\n" % (item_type, name, dep))
168      # We always include the dependency's exports, even if we do not need them
169      # to satisfy local imports.
170      exports |= dep_exports
171      system_symbols |= dep_system_symbols
172  item["exports"] = exports
173  item["system_symbols"] = system_symbols
174  imports -= exports | system_symbols
175  for symbol in imports:
176    for file_name in files:
177      if (file_name, symbol) in allowed_errors:
178         sys.stderr.write("Info:  ignoring %s imports %s\n\n" % (file_name, symbol))
179         continue
180      if symbol in _obj_files[file_name]["imports"]:
181        neededFile = _symbols_to_files.get(symbol)
182        if neededFile in dependencies.file_to_item:
183          neededItem = "but %s does not depend on %s (for %s)" % (name, dependencies.file_to_item[neededFile], neededFile)
184        else:
185          neededItem = "- is this a new system symbol?"
186        sys.stderr.write("Error: in %s %s: %s imports %s %s\n" %
187                         (item_type, name, file_name, symbol, neededItem))
188        _return_value = 1
189  del parents[-1]
190  return item
191
192def Process(root_path):
193  """Loads dependencies.txt, reads the libraries' .o files, and processes them.
194
195  Modifies dependencies.items: Recursively builds each item's system_symbols and exports.
196  """
197  global _ignored_symbols, _obj_files, _return_value
198  global _virtual_classes, _weak_destructors
199  dependencies.Load()
200  for name_and_item in iteritems(dependencies.items):
201    name = name_and_item[0]
202    item = name_and_item[1]
203    system_symbols = item.get("system_symbols")
204    if system_symbols:
205      for symbol in system_symbols:
206        _symbols_to_files[symbol] = name
207  for library_name in dependencies.libraries:
208    _ReadLibrary(root_path, library_name)
209  o_files_set = set(_obj_files.keys())
210  files_missing_from_deps = o_files_set - dependencies.files
211  files_missing_from_build = dependencies.files - o_files_set
212  if files_missing_from_deps:
213    sys.stderr.write("Error: files missing from dependencies.txt:\n%s\n" %
214                     sorted(files_missing_from_deps))
215    _return_value = 1
216  if files_missing_from_build:
217    sys.stderr.write("Error: files in dependencies.txt but not built:\n%s\n" %
218                     sorted(files_missing_from_build))
219    _return_value = 1
220  if not _return_value:
221    for library_name in dependencies.libraries:
222      _Resolve(library_name, [])
223  if not _return_value:
224    virtual_classes_with_weak_destructors = _virtual_classes & _weak_destructors
225    if virtual_classes_with_weak_destructors:
226      sys.stderr.write("Error: Some classes have virtual methods, and "
227                       "an implicit or inline destructor "
228                       "(see ICU ticket #8454 for details):\n%s\n" %
229                       sorted(virtual_classes_with_weak_destructors))
230      _return_value = 1
231
232def main():
233  global _return_value
234  if len(sys.argv) <= 1:
235    sys.exit(("Command line error: " +
236             "need one argument with the root path to the built ICU libraries/*.o files."))
237  Process(sys.argv[1])
238  if _ignored_symbols:
239    print("Info: ignored symbols:\n%s" % sorted(_ignored_symbols))
240  if not _return_value:
241    print("OK: Specified and actual dependencies match.")
242  else:
243    print("Error: There were errors, please fix them and re-run. Processing may have terminated abnormally.")
244  return _return_value
245
246if __name__ == "__main__":
247  sys.exit(main())
248