• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2014 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# This script computs the number of concurrent links we want to run in the build
7# as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
8
9import argparse
10import multiprocessing
11import os
12import re
13import subprocess
14import sys
15
16sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..'))
17import gn_helpers
18
19
20def _GetTotalMemoryInBytes():
21  if sys.platform in ('win32', 'cygwin'):
22    import ctypes
23
24    class MEMORYSTATUSEX(ctypes.Structure):
25      _fields_ = [
26          ("dwLength", ctypes.c_ulong),
27          ("dwMemoryLoad", ctypes.c_ulong),
28          ("ullTotalPhys", ctypes.c_ulonglong),
29          ("ullAvailPhys", ctypes.c_ulonglong),
30          ("ullTotalPageFile", ctypes.c_ulonglong),
31          ("ullAvailPageFile", ctypes.c_ulonglong),
32          ("ullTotalVirtual", ctypes.c_ulonglong),
33          ("ullAvailVirtual", ctypes.c_ulonglong),
34          ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
35      ]
36
37    stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
38    ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
39    return stat.ullTotalPhys
40  elif sys.platform.startswith('linux'):
41    if os.path.exists("/proc/meminfo"):
42      with open("/proc/meminfo") as meminfo:
43        memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
44        for line in meminfo:
45          match = memtotal_re.match(line)
46          if not match:
47            continue
48          return float(match.group(1)) * 2**10
49  elif sys.platform == 'darwin':
50    try:
51      return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
52    except Exception:
53      return 0
54  # TODO(scottmg): Implement this for other platforms.
55  return 0
56
57
58def _GetDefaultConcurrentLinks(per_link_gb, reserve_gb, thin_lto_type,
59                               secondary_per_link_gb, override_ram_in_gb):
60  explanation = []
61  explanation.append(
62      'per_link_gb={} reserve_gb={} secondary_per_link_gb={}'.format(
63          per_link_gb, reserve_gb, secondary_per_link_gb))
64  if override_ram_in_gb:
65    mem_total_gb = override_ram_in_gb
66  else:
67    mem_total_gb = float(_GetTotalMemoryInBytes()) / 2**30
68  adjusted_mem_total_gb = max(0, mem_total_gb - reserve_gb)
69
70  # Ensure that there is at least as many links allocated for the secondary as
71  # there is for the primary. The secondary link usually uses fewer gbs.
72  mem_cap = int(
73      max(1, adjusted_mem_total_gb / (per_link_gb + secondary_per_link_gb)))
74
75  try:
76    cpu_count = multiprocessing.cpu_count()
77  except:
78    cpu_count = 1
79
80  # A local LTO links saturate all cores, but only for some amount of the link.
81  # Goma LTO runs LTO codegen on goma, only run one of these tasks at once.
82  cpu_cap = cpu_count
83  if thin_lto_type is not None:
84    if thin_lto_type == 'goma':
85      cpu_cap = 1
86    else:
87      assert thin_lto_type == 'local'
88      cpu_cap = min(cpu_count, 6)
89
90  explanation.append(
91      'cpu_count={} cpu_cap={} mem_total_gb={:.1f}GiB adjusted_mem_total_gb={:.1f}GiB'
92      .format(cpu_count, cpu_cap, mem_total_gb, adjusted_mem_total_gb))
93
94  num_links = min(mem_cap, cpu_cap)
95  if num_links == cpu_cap:
96    if cpu_cap == cpu_count:
97      reason = 'cpu_count'
98    else:
99      reason = 'cpu_cap (thinlto)'
100  else:
101    reason = 'RAM'
102
103  # static link see too many open files if we have many concurrent links.
104  # ref: http://b/233068481
105  if num_links > 30:
106    num_links = 30
107    reason = 'nofile'
108
109  explanation.append('concurrent_links={}  (reason: {})'.format(
110      num_links, reason))
111
112  # Use remaining RAM for a secondary pool if needed.
113  if secondary_per_link_gb:
114    mem_remaining = adjusted_mem_total_gb - num_links * per_link_gb
115    secondary_size = int(max(0, mem_remaining / secondary_per_link_gb))
116    if secondary_size > cpu_count:
117      secondary_size = cpu_count
118      reason = 'cpu_count'
119    else:
120      reason = 'mem_remaining={:.1f}GiB'.format(mem_remaining)
121    explanation.append('secondary_size={} (reason: {})'.format(
122        secondary_size, reason))
123  else:
124    secondary_size = 0
125
126  return num_links, secondary_size, explanation
127
128
129def main():
130  parser = argparse.ArgumentParser()
131  parser.add_argument('--mem_per_link_gb', type=int, default=8)
132  parser.add_argument('--reserve_mem_gb', type=int, default=0)
133  parser.add_argument('--secondary_mem_per_link', type=int, default=0)
134  parser.add_argument('--override-ram-in-gb-for-testing', type=float, default=0)
135  parser.add_argument('--thin-lto')
136  options = parser.parse_args()
137
138  primary_pool_size, secondary_pool_size, explanation = (
139      _GetDefaultConcurrentLinks(options.mem_per_link_gb,
140                                 options.reserve_mem_gb, options.thin_lto,
141                                 options.secondary_mem_per_link,
142                                 options.override_ram_in_gb_for_testing))
143  if options.override_ram_in_gb_for_testing:
144    print('primary={} secondary={} explanation={}'.format(
145        primary_pool_size, secondary_pool_size, explanation))
146  else:
147    sys.stdout.write(
148        gn_helpers.ToGNString({
149            'primary_pool_size': primary_pool_size,
150            'secondary_pool_size': secondary_pool_size,
151            'explanation': explanation,
152        }))
153  return 0
154
155
156if __name__ == '__main__':
157  sys.exit(main())
158