• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2017 Google Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16################################################################################
17
18from __future__ import print_function
19import argparse
20import imp
21import os
22import multiprocessing
23import resource
24import shutil
25import subprocess
26import tempfile
27
28import apt
29from apt import debfile
30
31from packages import package
32import wrapper_utils
33
34SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
35PACKAGES_DIR = os.path.join(SCRIPT_DIR, 'packages')
36
37TRACK_ORIGINS_ARG = '-fsanitize-memory-track-origins='
38
39INJECTED_ARGS = [
40    '-fsanitize=memory',
41    '-fsanitize-recover=memory',
42    '-fPIC',
43    '-fno-omit-frame-pointer',
44]
45
46
47class MSanBuildException(Exception):
48  """Base exception."""
49
50
51def GetTrackOriginsFlag():
52  """Get the track origins flag."""
53  if os.getenv('MSAN_NO_TRACK_ORIGINS'):
54    return TRACK_ORIGINS_ARG + '0'
55
56  return TRACK_ORIGINS_ARG + '2'
57
58
59def GetInjectedFlags():
60  return INJECTED_ARGS + [GetTrackOriginsFlag()]
61
62
63def SetUpEnvironment(work_dir):
64  """Set up build environment."""
65  env = {}
66  env['REAL_CLANG_PATH'] = subprocess.check_output(['which', 'clang']).strip()
67  print('Real clang at', env['REAL_CLANG_PATH'])
68  compiler_wrapper_path = os.path.join(SCRIPT_DIR, 'compiler_wrapper.py')
69
70  # Symlink binaries into TMP/bin
71  bin_dir = os.path.join(work_dir, 'bin')
72  os.mkdir(bin_dir)
73
74  dpkg_host_architecture = wrapper_utils.DpkgHostArchitecture()
75  wrapper_utils.CreateSymlinks(
76      compiler_wrapper_path, bin_dir, [
77          'clang',
78          'clang++',
79          # Not all build rules respect $CC/$CXX, so make additional symlinks.
80          'gcc',
81          'g++',
82          'cc',
83          'c++',
84          dpkg_host_architecture + '-gcc',
85          dpkg_host_architecture + '-g++',
86      ])
87
88  env['CC'] = os.path.join(bin_dir, 'clang')
89  env['CXX'] = os.path.join(bin_dir, 'clang++')
90
91  MSAN_OPTIONS = ' '.join(GetInjectedFlags())
92
93  # We don't use nostrip because some build rules incorrectly break when it is
94  # passed. Instead we install our own no-op strip binaries.
95  env['DEB_BUILD_OPTIONS'] = ('nocheck parallel=%d' %
96                              multiprocessing.cpu_count())
97  env['DEB_CFLAGS_APPEND'] = MSAN_OPTIONS
98  env['DEB_CXXFLAGS_APPEND'] = MSAN_OPTIONS + ' -stdlib=libc++'
99  env['DEB_CPPFLAGS_APPEND'] = MSAN_OPTIONS
100  env['DEB_LDFLAGS_APPEND'] = MSAN_OPTIONS
101  env['DPKG_GENSYMBOLS_CHECK_LEVEL'] = '0'
102
103  # debian/rules can set DPKG_GENSYMBOLS_CHECK_LEVEL explicitly, so override it.
104  gen_symbols_wrapper = (
105        '#!/bin/sh\n'
106        'export DPKG_GENSYMBOLS_CHECK_LEVEL=0\n'
107        '/usr/bin/dpkg-gensymbols "$@"\n')
108
109  wrapper_utils.InstallWrapper(bin_dir, 'dpkg-gensymbols',
110                               gen_symbols_wrapper)
111
112  # Install no-op strip binaries.
113  no_op_strip = ('#!/bin/sh\n'
114                 'exit 0\n')
115  wrapper_utils.InstallWrapper(
116      bin_dir, 'strip', no_op_strip,
117      [dpkg_host_architecture + '-strip'])
118
119  env['PATH'] = bin_dir + ':' + os.environ['PATH']
120
121  # nocheck doesn't disable override_dh_auto_test. So we have this hack to try
122  # to disable "make check" or "make test" invocations.
123  make_wrapper = (
124      '#!/bin/bash\n'
125      'if [ "$1" = "test" ] || [ "$1" = "check" ]; then\n'
126      '  exit 0\n'
127      'fi\n'
128      '/usr/bin/make "$@"\n')
129  wrapper_utils.InstallWrapper(bin_dir, 'make',
130                               make_wrapper)
131
132  # Prevent entire build from failing because of bugs/uninstrumented in tools
133  # that are part of the build.
134  msan_log_dir = os.path.join(work_dir, 'msan')
135  os.mkdir(msan_log_dir)
136  msan_log_path = os.path.join(msan_log_dir, 'log')
137  env['MSAN_OPTIONS'] = (
138      'halt_on_error=0:exitcode=0:report_umrs=0:log_path=' + msan_log_path)
139
140  # Increase maximum stack size to prevent tests from failing.
141  limit = 128 * 1024 * 1024
142  resource.setrlimit(resource.RLIMIT_STACK, (limit, limit))
143  return env
144
145
146def FindPackageDebs(package_name, work_directory):
147  """Find package debs."""
148  deb_paths = []
149  cache = apt.Cache()
150
151  for filename in os.listdir(work_directory):
152    file_path = os.path.join(work_directory, filename)
153    if not file_path.endswith('.deb'):
154      continue
155
156    # Matching package name.
157    deb = debfile.DebPackage(file_path)
158    if deb.pkgname == package_name:
159      deb_paths.append(file_path)
160      continue
161
162    # Also include -dev packages that depend on the runtime package.
163    pkg = cache[deb.pkgname]
164    if pkg.section != 'libdevel' and pkg.section != 'universe/libdevel':
165      continue
166
167    # But ignore -dbg packages.
168    if deb.pkgname.endswith('-dbg'):
169      continue
170
171    for dependency in deb.depends:
172      if any(dep[0] == package_name for dep in dependency):
173        deb_paths.append(file_path)
174        break
175
176  return deb_paths
177
178
179def ExtractLibraries(deb_paths, work_directory, output_directory):
180  """Extract libraries from .deb packages."""
181  extract_directory = os.path.join(work_directory, 'extracted')
182  if os.path.exists(extract_directory):
183    shutil.rmtree(extract_directory, ignore_errors=True)
184
185  os.mkdir(extract_directory)
186
187  for deb_path in deb_paths:
188    subprocess.check_call(['dpkg-deb', '-x', deb_path, extract_directory])
189
190  extracted = []
191  for root, _, filenames in os.walk(extract_directory):
192    if 'libx32' in root or 'lib32' in root:
193      continue
194
195    for filename in filenames:
196      if (not filename.endswith('.so') and '.so.' not in filename and
197          not filename.endswith('.a') and '.a' not in filename):
198        continue
199
200      file_path = os.path.join(root, filename)
201      rel_file_path = os.path.relpath(file_path, extract_directory)
202      rel_directory = os.path.dirname(rel_file_path)
203
204      target_dir = os.path.join(output_directory, rel_directory)
205      if not os.path.exists(target_dir):
206        os.makedirs(target_dir)
207
208      target_file_path = os.path.join(output_directory, rel_file_path)
209      extracted.append(target_file_path)
210
211      if os.path.lexists(target_file_path):
212        os.remove(target_file_path)
213
214      if os.path.islink(file_path):
215        link_path = os.readlink(file_path)
216        if os.path.isabs(link_path):
217          # Make absolute links relative.
218          link_path = os.path.relpath(
219              link_path, os.path.join('/', rel_directory))
220
221        os.symlink(link_path, target_file_path)
222      else:
223        shutil.copy2(file_path, target_file_path)
224
225  return extracted
226
227
228def GetPackage(package_name):
229  apt_cache = apt.Cache()
230  version = apt_cache[package_name].candidate
231  source_name = version.source_name
232  local_source_name = source_name.replace('.', '_')
233
234  custom_package_path = os.path.join(PACKAGES_DIR, local_source_name) + '.py'
235  if not os.path.exists(custom_package_path):
236    print('Using default package build steps.')
237    return package.Package(source_name, version)
238
239  print('Using custom package build steps.')
240  module = imp.load_source('packages.' + local_source_name, custom_package_path)
241  return module.Package(version)
242
243
244def PatchRpath(path, output_directory):
245  """Patch rpath to be relative to $ORIGIN."""
246  try:
247    rpaths = subprocess.check_output(
248        ['patchelf', '--print-rpath', path]).strip()
249  except subprocess.CalledProcessError:
250    return
251
252  if not rpaths:
253    return
254
255  processed_rpath = []
256  rel_directory = os.path.join(
257      '/', os.path.dirname(os.path.relpath(path, output_directory)))
258
259  for rpath in rpaths.split(':'):
260    if '$ORIGIN' in rpath:
261      # Already relative.
262      processed_rpath.append(rpath)
263      continue
264
265    processed_rpath.append(os.path.join(
266        '$ORIGIN',
267        os.path.relpath(rpath, rel_directory)))
268
269  processed_rpath = ':'.join(processed_rpath)
270  print('Patching rpath for', path, 'to', processed_rpath)
271  subprocess.check_call(
272      ['patchelf', '--force-rpath', '--set-rpath',
273       processed_rpath, path])
274
275
276def _CollectDependencies(apt_cache, pkg, cache, dependencies):
277  """Collect dependencies that need to be built."""
278  C_OR_CXX_DEPS = [
279      'libc++1',
280      'libc6',
281      'libc++abi1',
282      'libgcc1',
283      'libstdc++6',
284  ]
285
286  BLACKLISTED_PACKAGES = [
287      'libcapnp-0.5.3',  # fails to compile on newer clang.
288      'libllvm5.0',
289      'libmircore1',
290      'libmircommon7',
291      'libmirclient9',
292      'libmirprotobuf3',
293      'multiarch-support',
294  ]
295
296  if pkg.name in BLACKLISTED_PACKAGES:
297    return False
298
299  if pkg.section != 'libs' and pkg.section != 'universe/libs':
300    return False
301
302  if pkg.name in C_OR_CXX_DEPS:
303    return True
304
305  is_c_or_cxx = False
306  for dependency in pkg.candidate.dependencies:
307    dependency = dependency[0]
308
309    if dependency.name in cache:
310      is_c_or_cxx |= cache[dependency.name]
311    else:
312      is_c_or_cxx |= _CollectDependencies(apt_cache, apt_cache[dependency.name],
313                                          cache, dependencies)
314  if is_c_or_cxx:
315    dependencies.append(pkg.name)
316
317  cache[pkg.name] = is_c_or_cxx
318  return is_c_or_cxx
319
320
321def GetBuildList(package_name):
322  """Get list of packages that need to be built including dependencies."""
323  apt_cache = apt.Cache()
324  pkg = apt_cache[package_name]
325
326  dependencies = []
327  _CollectDependencies(apt_cache, pkg, {}, dependencies)
328  return dependencies
329
330
331class MSanBuilder(object):
332  """MSan builder."""
333
334  def __init__(self, debug=False, log_path=None, work_dir=None, no_track_origins=False):
335    self.debug = debug
336    self.log_path = log_path
337    self.work_dir = work_dir
338    self.no_track_origins = no_track_origins
339    self.env = None
340
341  def __enter__(self):
342    if not self.work_dir:
343      self.work_dir = tempfile.mkdtemp(dir=self.work_dir)
344
345    if os.path.exists(self.work_dir):
346      shutil.rmtree(self.work_dir, ignore_errors=True)
347
348    os.makedirs(self.work_dir)
349    self.env = SetUpEnvironment(self.work_dir)
350
351    if self.debug and self.log_path:
352      self.env['WRAPPER_DEBUG_LOG_PATH'] = self.log_path
353
354    if self.no_track_origins:
355      self.env['MSAN_NO_TRACK_ORIGINS'] = '1'
356
357    return self
358
359  def __exit__(self, exc_type, exc_value, traceback):
360    if not self.debug:
361      shutil.rmtree(self.work_dir, ignore_errors=True)
362
363  def Build(self, package_name, output_directory, create_subdirs=False):
364    """Build the package and write results into the output directory."""
365    deb_paths = FindPackageDebs(package_name, self.work_dir)
366    if deb_paths:
367      print('Source package already built for', package_name)
368    else:
369      pkg = GetPackage(package_name)
370
371      pkg.InstallBuildDeps()
372      source_directory = pkg.DownloadSource(self.work_dir)
373      print('Source downloaded to', source_directory)
374
375      # custom bin directory for custom build scripts to write wrappers.
376      custom_bin_dir = os.path.join(self.work_dir, package_name + '_bin')
377      os.mkdir(custom_bin_dir)
378      env = self.env.copy()
379      env['PATH'] = custom_bin_dir + ':' + env['PATH']
380
381      pkg.Build(source_directory, env, custom_bin_dir)
382      shutil.rmtree(custom_bin_dir, ignore_errors=True)
383
384      deb_paths = FindPackageDebs(package_name, self.work_dir)
385
386    if not deb_paths:
387      raise MSanBuildException('Failed to find .deb packages.')
388
389    print('Extracting', ' '.join(deb_paths))
390
391    if create_subdirs:
392      extract_directory = os.path.join(output_directory, package_name)
393    else:
394      extract_directory = output_directory
395
396    extracted_paths = ExtractLibraries(deb_paths, self.work_dir,
397                                       extract_directory)
398    for extracted_path in extracted_paths:
399      if not os.path.islink(extracted_path):
400        PatchRpath(extracted_path, extract_directory)
401
402
403def main():
404  parser = argparse.ArgumentParser('msan_build.py', description='MSan builder.')
405  parser.add_argument('package_names', nargs='+', help='Name of the packages.')
406  parser.add_argument('output_dir', help='Output directory.')
407  parser.add_argument('--create-subdirs', action='store_true',
408                      help=('Create subdirectories in the output '
409                            'directory for each package.'))
410  parser.add_argument('--work-dir', help='Work directory.')
411  parser.add_argument('--no-build-deps', action='store_true',
412                      help='Don\'t build dependencies.')
413  parser.add_argument('--debug', action='store_true', help='Enable debug mode.')
414  parser.add_argument('--log-path', help='Log path for debugging.')
415  parser.add_argument('--no-track-origins',
416                      action='store_true',
417                      help='Build with -fsanitize-memory-track-origins=0.')
418  args = parser.parse_args()
419
420  if args.no_track_origins:
421    os.environ['MSAN_NO_TRACK_ORIGINS'] = '1'
422
423  if not os.path.exists(args.output_dir):
424    os.makedirs(args.output_dir)
425
426  if args.no_build_deps:
427    package_names = args.package_names
428  else:
429    all_packages = set()
430    package_names = []
431
432    # Get list of packages to build, including all dependencies.
433    for package_name in args.package_names:
434      for dep in GetBuildList(package_name):
435        if dep in all_packages:
436          continue
437
438        if args.create_subdirs:
439          os.mkdir(os.path.join(args.output_dir, dep))
440
441        all_packages.add(dep)
442        package_names.append(dep)
443
444  print('Going to build:')
445  for package_name in package_names:
446    print('\t', package_name)
447
448  with MSanBuilder(debug=args.debug, log_path=args.log_path,
449                   work_dir=args.work_dir,
450                   no_track_origins=args.no_track_origins) as builder:
451    for package_name in package_names:
452      builder.Build(package_name, args.output_dir, args.create_subdirs)
453
454
455if __name__ == '__main__':
456  main()
457