• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3# NOTE: This script requires python 3.
4
5"""Script to generate Chromium's Abseil .def files at roll time.
6
7This script generates //third_party/abseil-app/absl/symbols_*.def at Abseil
8roll time.
9
10Since Abseil doesn't export symbols, Chromium is forced to consider all
11Abseil's symbols as publicly visible. On POSIX it is possible to use
12-fvisibility=default but on Windows a .def file with all the symbols
13is needed.
14
15Unless you are on a Windows machine, you need to set up your Chromium
16checkout for cross-compilation by following the instructions at
17https://chromium.googlesource.com/chromium/src.git/+/main/docs/win_cross.md.
18If you are on Windows, you may need to tweak this script to run, e.g. by
19changing "gn" to "gn.bat", changing "llvm-nm" to the name of your copy of
20llvm-nm, etc.
21"""
22
23import fnmatch
24import logging
25import os
26import re
27import subprocess
28import sys
29import tempfile
30import time
31
32# Matches a mangled symbol that has 'absl' in it, this should be a good
33# enough heuristic to select Abseil symbols to list in the .def file.
34ABSL_SYM_RE = re.compile(r'0* [BT] (?P<symbol>(\?+)[^\?].*absl.*)')
35if sys.platform == 'win32':
36  # Typical dumpbin /symbol lines look like this:
37  # 04B 0000000C SECT14 notype       Static       | ?$S1@?1??SetCurrent
38  # ThreadIdentity@base_internal@absl@@YAXPAUThreadIdentity@12@P6AXPAX@Z@Z@4IA
39  #  (unsigned int `void __cdecl absl::base_internal::SetCurrentThreadIdentity...
40  # We need to start on "| ?" and end on the first " (" (stopping on space would
41  # also work).
42  # This regex is identical inside the () characters except for the ? after .*,
43  # which is needed to prevent greedily grabbing the undecorated version of the
44  # symbols.
45  ABSL_SYM_RE = '.*External     \| (?P<symbol>(\?+)[^\?].*?absl.*?) \(.*'
46  # Typical exported symbols in dumpbin /directives look like:
47  #    /EXPORT:?kHexChar@numbers_internal@absl@@3QBDB,DATA
48  ABSL_EXPORTED_RE = '.*/EXPORT:(.*),.*'
49
50
51def _DebugOrRelease(is_debug):
52  return 'dbg' if is_debug else 'rel'
53
54
55def _GenerateDefFile(cpu, is_debug, extra_gn_args=[], suffix=None):
56  """Generates a .def file for the absl component build on the specified CPU."""
57  if extra_gn_args:
58    assert suffix != None, 'suffix is needed when extra_gn_args is used'
59
60  flavor = _DebugOrRelease(is_debug)
61  gn_args = [
62      'ffmpeg_branding = "Chrome"',
63      'is_component_build = true',
64      'is_debug = {}'.format(str(is_debug).lower()),
65      'proprietary_codecs = true',
66      'symbol_level = 0',
67      'target_cpu = "{}"'.format(cpu),
68      'target_os = "win"',
69  ]
70  gn_args.extend(extra_gn_args)
71
72  gn = 'gn'
73  autoninja = 'autoninja'
74  symbol_dumper = ['third_party/llvm-build/Release+Asserts/bin/llvm-nm']
75  if sys.platform == 'win32':
76    gn = 'gn.bat'
77    autoninja = 'autoninja.bat'
78    symbol_dumper = ['dumpbin', '/symbols']
79    import shutil
80    if not shutil.which('dumpbin'):
81      logging.error('dumpbin not found. Run tools\win\setenv.bat.')
82      exit(1)
83  with tempfile.TemporaryDirectory() as out_dir:
84    logging.info('[%s - %s] Creating tmp out dir in %s', cpu, flavor, out_dir)
85    subprocess.check_call([gn, 'gen', out_dir, '--args=' + ' '.join(gn_args)],
86                          cwd=os.getcwd())
87    logging.info('[%s - %s] gn gen completed', cpu, flavor)
88    subprocess.check_call(
89        [autoninja, '-C', out_dir, 'third_party/abseil-cpp:absl_component_deps'],
90        cwd=os.getcwd())
91    logging.info('[%s - %s] autoninja completed', cpu, flavor)
92
93    obj_files = []
94    for root, _dirnames, filenames in os.walk(
95        os.path.join(out_dir, 'obj', 'third_party', 'abseil-cpp')):
96      matched_files = fnmatch.filter(filenames, '*.obj')
97      obj_files.extend((os.path.join(root, f) for f in matched_files))
98
99    logging.info('[%s - %s] Found %d object files.', cpu, flavor, len(obj_files))
100
101    absl_symbols = set()
102    dll_exports = set()
103    if sys.platform == 'win32':
104      for f in obj_files:
105        # Track all of the functions exported with __declspec(dllexport) and
106        # don't list them in the .def file - double-exports are not allowed. The
107        # error is "lld-link: error: duplicate /export option".
108        exports_out = subprocess.check_output(['dumpbin', '/directives', f], cwd=os.getcwd())
109        for line in exports_out.splitlines():
110          line = line.decode('utf-8')
111          match = re.match(ABSL_EXPORTED_RE, line)
112          if match:
113            dll_exports.add(match.groups()[0])
114    for f in obj_files:
115      stdout = subprocess.check_output(symbol_dumper + [f], cwd=os.getcwd())
116      for line in stdout.splitlines():
117        try:
118          line = line.decode('utf-8')
119        except UnicodeDecodeError:
120          # Due to a dumpbin bug there are sometimes invalid utf-8 characters in
121          # the output. This only happens on an unimportant line so it can
122          # safely and silently be skipped.
123          # https://developercommunity.visualstudio.com/content/problem/1091330/dumpbin-symbols-produces-randomly-wrong-output-on.html
124          continue
125        match = re.match(ABSL_SYM_RE, line)
126        if match:
127          symbol = match.group('symbol')
128          assert symbol.count(' ') == 0, ('Regex matched too much, probably got '
129                                          'undecorated name as well')
130          # Avoid getting names exported with dllexport, to avoid
131          # "lld-link: error: duplicate /export option" on symbols such as:
132          # ?kHexChar@numbers_internal@absl@@3QBDB
133          if symbol in dll_exports:
134            continue
135          # Avoid to export deleting dtors since they trigger
136          # "lld-link: error: export of deleting dtor" linker errors, see
137          # crbug.com/1201277.
138          if symbol.startswith('??_G'):
139            continue
140          absl_symbols.add(symbol)
141
142    logging.info('[%s - %s] Found %d absl symbols.', cpu, flavor, len(absl_symbols))
143
144    if extra_gn_args:
145      def_file = os.path.join('third_party', 'abseil-cpp',
146                              'symbols_{}_{}_{}.def'.format(cpu, flavor, suffix))
147    else:
148      def_file = os.path.join('third_party', 'abseil-cpp',
149                             'symbols_{}_{}.def'.format(cpu, flavor))
150
151    with open(def_file, 'w', newline='') as f:
152      f.write('EXPORTS\n')
153      for s in sorted(absl_symbols):
154        f.write('    {}\n'.format(s))
155
156    # Hack, it looks like there is a race in the directory cleanup.
157    time.sleep(10)
158
159  logging.info('[%s - %s] .def file successfully generated.', cpu, flavor)
160
161
162if __name__ == '__main__':
163  logging.getLogger().setLevel(logging.INFO)
164
165  if sys.version_info.major == 2:
166    logging.error('This script requires Python 3.')
167    exit(1)
168
169  if not os.getcwd().endswith('src') or not os.path.exists('chrome/browser'):
170    logging.error('Run this script from a chromium/src/ directory.')
171    exit(1)
172
173  _GenerateDefFile('x86', True)
174  _GenerateDefFile('x86', False)
175  _GenerateDefFile('x64', True)
176  _GenerateDefFile('x64', False)
177  _GenerateDefFile('x64', False, ['is_asan = true'], 'asan')
178  _GenerateDefFile('arm64', True)
179  _GenerateDefFile('arm64', False)
180