• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# Copyright 2014 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# This script computes the number of concurrent links we want to run
8# in the build as a function of machine spec. It's based
9# on GetDefaultConcurrentLinks in GYP.
10
11import multiprocessing
12import optparse
13import os
14import re
15import subprocess
16import sys
17
18
19def _get_total_memory_in_bytes():
20    if sys.platform in ('win32', 'cygwin'):
21        import ctypes
22
23        class MEMORYSTATUSEX(ctypes.Structure):
24            _fields_ = [
25                ("dwLength", ctypes.c_ulong),
26                ("dwMemoryLoad", ctypes.c_ulong),
27                ("ullTotalPhys", ctypes.c_ulonglong),
28                ("ullAvailPhys", ctypes.c_ulonglong),
29                ("ullTotalPageFile", ctypes.c_ulonglong),
30                ("ullAvailPageFile", ctypes.c_ulonglong),
31                ("ullTotalVirtual", ctypes.c_ulonglong),
32                ("ullAvailVirtual", ctypes.c_ulonglong),
33                ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
34            ]
35
36        stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
37        ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
38        return stat.ullTotalPhys
39    elif sys.platform.startswith('linux'):
40        if os.path.exists("/proc/meminfo"):
41            with open("/proc/meminfo") as meminfo:
42                memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
43                for line in meminfo:
44                    match = memtotal_re.match(line)
45                    if not match:
46                        continue
47                    return float(match.group(1)) * 2**10
48    elif sys.platform == 'darwin':
49        try:
50            return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
51        except Exception:
52            return 0
53    return 0
54
55
56def _get_default_concurrent_links(mem_per_link_gb, reserve_mem_gb):
57    mem_total_bytes = _get_total_memory_in_bytes()
58    mem_total_bytes = max(0, mem_total_bytes - reserve_mem_gb * 2**30)
59    num_concurrent_links = int(
60        max(1, mem_total_bytes / mem_per_link_gb / 2**30))
61    hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
62
63    try:
64        cpu_cap = multiprocessing.cpu_count()
65    except: # noqa E722
66        cpu_cap = 1
67
68    return min(num_concurrent_links, hard_cap, cpu_cap)
69
70
71def main():
72    parser = optparse.OptionParser()
73    parser.add_option('--mem_per_link_gb',
74                      action="store",
75                      type="int",
76                      default=8)
77    parser.add_option('--reserve_mem_gb',
78                      action="store",
79                      type="int",
80                      default=0)
81    parser.disable_interspersed_args()
82    options, _ = parser.parse_args()
83
84    print(
85        _get_default_concurrent_links(options.mem_per_link_gb,
86                                      options.reserve_mem_gb))
87    return 0
88
89
90if __name__ == '__main__':
91    sys.exit(main())
92