• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5# This script computs the number of concurrent links we want to run in the build
6# as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
7
8import optparse
9import os
10import re
11import subprocess
12import sys
13
14def _GetDefaultConcurrentLinks(is_lto):
15  # Inherit the legacy environment variable for people that have set it in GYP.
16  pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0))
17  if pool_size:
18    return pool_size
19
20  if sys.platform in ('win32', 'cygwin'):
21    import ctypes
22
23    class MEMORYSTATUSEX(ctypes.Structure):
24      _fields_ = [
25        ("dwLength", ctypes.c_ulong),
26        ("dwMemoryLoad", ctypes.c_ulong),
27        ("ullTotalPhys", ctypes.c_ulonglong),
28        ("ullAvailPhys", ctypes.c_ulonglong),
29        ("ullTotalPageFile", ctypes.c_ulonglong),
30        ("ullAvailPageFile", ctypes.c_ulonglong),
31        ("ullTotalVirtual", ctypes.c_ulonglong),
32        ("ullAvailVirtual", ctypes.c_ulonglong),
33        ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
34      ]
35
36    stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
37    ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
38
39    # VS 2015 uses 20% more working set than VS 2013 and can consume all RAM
40    # on a 64 GB machine.
41    mem_limit = max(1, stat.ullTotalPhys / (5 * (2 ** 30)))  # total / 5GB
42    hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
43    return min(mem_limit, hard_cap)
44  elif sys.platform.startswith('linux'):
45    if os.path.exists("/proc/meminfo"):
46      with open("/proc/meminfo") as meminfo:
47        memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
48        for line in meminfo:
49          match = memtotal_re.match(line)
50          if not match:
51            continue
52          mem_total_gb = float(match.group(1)) / (2 ** 20)
53          # Allow 8Gb per link on Linux because Gold is quite memory hungry
54          mem_per_link_gb = 8
55          if is_lto:
56            mem_total_gb -= 10 # Reserve
57            # For LTO builds the RAM requirements are even higher
58            mem_per_link_gb = 16
59          return int(max(1, mem_total_gb / mem_per_link_gb))
60    return 1
61  elif sys.platform == 'darwin':
62    try:
63      avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
64      # A static library debug build of Chromium's unit_tests takes ~2.7GB, so
65      # 4GB per ld process allows for some more bloat.
66      return max(1, avail_bytes / (4 * (2 ** 30)))  # total / 4GB
67    except Exception:
68      return 1
69  else:
70    # TODO(scottmg): Implement this for other platforms.
71    return 1
72
73def main():
74  parser = optparse.OptionParser()
75  parser.add_option('--lto', action="store_true", default=False,
76                    help='This is an LTO build with higher memory requirements')
77  parser.disable_interspersed_args()
78  options, args = parser.parse_args()
79
80  print _GetDefaultConcurrentLinks(is_lto=options.lto)
81  return 0
82
83if __name__ == '__main__':
84  sys.exit(main())
85