• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright (c) 2024 Huawei Device Co., Ltd.
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# This script computes the number of concurrent links we want to run
17# in the build as a function of machine spec. It's based
18# on GetDefaultConcurrentLinks in GYP.
19
20import multiprocessing
21import optparse
22import os
23import re
24import subprocess
25import sys
26
27
28def _get_total_memory_in_bytes():
29    if sys.platform in ('win32', 'cygwin'):
30        import ctypes
31
32        class MEMORYSTATUSEX(ctypes.Structure):
33            _fields_ = [
34                ("dwLength", ctypes.c_ulong),
35                ("dwMemoryLoad", ctypes.c_ulong),
36                ("ullTotalPhys", ctypes.c_ulonglong),
37                ("ullAvailPhys", ctypes.c_ulonglong),
38                ("ullTotalPageFile", ctypes.c_ulonglong),
39                ("ullAvailPageFile", ctypes.c_ulonglong),
40                ("ullTotalVirtual", ctypes.c_ulonglong),
41                ("ullAvailVirtual", ctypes.c_ulonglong),
42                ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
43            ]
44
45        stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
46        ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
47        return stat.ullTotalPhys
48    elif sys.platform.startswith('linux'):
49        if os.path.exists("/proc/meminfo"):
50            with open("/proc/meminfo") as meminfo:
51                memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
52                for line in meminfo:
53                    match = memtotal_re.match(line)
54                    if not match:
55                        continue
56                    return float(match.group(1)) * 2**10
57    elif sys.platform == 'darwin':
58        try:
59            return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
60        except Exception:
61            return 0
62    return 0
63
64
65def _get_default_concurrent_links(mem_per_link_gb, reserve_mem_gb):
66    mem_total_bytes = _get_total_memory_in_bytes()
67    mem_total_bytes = max(0, mem_total_bytes - reserve_mem_gb * 2**30)
68    num_concurrent_links = int(
69        max(1, mem_total_bytes / mem_per_link_gb / 2**30))
70    hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
71
72    try:
73        cpu_cap = multiprocessing.cpu_count()
74    except: # noqa E722
75        cpu_cap = 1
76
77    return min(num_concurrent_links, hard_cap, cpu_cap)
78
79
80def main():
81    parser = optparse.OptionParser()
82    parser.add_option('--mem_per_link_gb',
83                      action="store",
84                      type="int",
85                      default=8)
86    parser.add_option('--reserve_mem_gb',
87                      action="store",
88                      type="int",
89                      default=0)
90    parser.disable_interspersed_args()
91    options, _ = parser.parse_args()
92
93    print(
94        _get_default_concurrent_links(options.mem_per_link_gb,
95                                      options.reserve_mem_gb))
96    return 0
97
98
99if __name__ == '__main__':
100    sys.exit(main())
101