• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2017 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""\
6Convenience wrapper for compiling V8 with gn/ninja and running tests.
7Sets up build output directories if they don't exist.
8Produces simulator builds for non-Intel target architectures.
9Uses Goma by default if it is detected (at output directory setup time).
10Expects to be run from the root of a V8 checkout.
11
12Usage:
13    gm.py [<arch>].[<mode>[-<suffix>]].[<target>] [testname...] [--flag]
14
15All arguments are optional. Most combinations should work, e.g.:
16    gm.py ia32.debug x64.release x64.release-my-custom-opts d8
17    gm.py android_arm.release.check --progress=verbose
18    gm.py x64 mjsunit/foo cctest/test-bar/*
19
20Flags are passed unchanged to the test runner. They must start with -- and must
21not contain spaces.
22"""
23# See HELP below for additional documentation.
24
25from __future__ import print_function
26import errno
27import os
28import platform
29import re
30import subprocess
31import sys
32
33USE_PTY = "linux" in sys.platform
34if USE_PTY:
35  import pty
36
37BUILD_TARGETS_TEST = ["d8", "bigint_shell", "cctest", "inspector-test",
38                      "unittests", "wasm_api_tests"]
39BUILD_TARGETS_ALL = ["all"]
40
41# All arches that this script understands.
42ARCHES = ["ia32", "x64", "arm", "arm64", "mipsel", "mips64el", "ppc", "ppc64",
43          "riscv64", "s390", "s390x", "android_arm", "android_arm64", "loong64",
44          "fuchsia_x64", "fuchsia_arm64"]
45# Arches that get built/run when you don't specify any.
46DEFAULT_ARCHES = ["ia32", "x64", "arm", "arm64"]
47# Modes that this script understands.
48MODES = ["release", "debug", "optdebug"]
49# Modes that get built/run when you don't specify any.
50DEFAULT_MODES = ["release", "debug"]
51# Build targets that can be manually specified.
52TARGETS = ["d8", "cctest", "unittests", "v8_fuzzers", "wasm_api_tests", "wee8",
53           "mkgrokdump", "generate-bytecode-expectations", "inspector-test",
54           "bigint_shell"]
55# Build targets that get built when you don't specify any (and specified tests
56# don't imply any other targets).
57DEFAULT_TARGETS = ["d8"]
58# Tests that run-tests.py would run by default that can be run with
59# BUILD_TARGETS_TESTS.
60DEFAULT_TESTS = ["cctest", "debugger", "intl", "message", "mjsunit",
61                 "unittests"]
62# These can be suffixed to any <arch>.<mode> combo, or used standalone,
63# or used as global modifiers (affecting all <arch>.<mode> combos).
64ACTIONS = {
65    "all": {
66        "targets": BUILD_TARGETS_ALL,
67        "tests": [],
68        "clean": False
69    },
70    "tests": {
71        "targets": BUILD_TARGETS_TEST,
72        "tests": [],
73        "clean": False
74    },
75    "check": {
76        "targets": BUILD_TARGETS_TEST,
77        "tests": DEFAULT_TESTS,
78        "clean": False
79    },
80    "checkall": {
81        "targets": BUILD_TARGETS_ALL,
82        "tests": ["ALL"],
83        "clean": False
84    },
85    "clean": {
86        "targets": [],
87        "tests": [],
88        "clean": True
89    },
90}
91
92HELP = """<arch> can be any of: %(arches)s
93<mode> can be any of: %(modes)s
94<target> can be any of:
95 - %(targets)s (build respective binary)
96 - all (build all binaries)
97 - tests (build test binaries)
98 - check (build test binaries, run most tests)
99 - checkall (build all binaries, run more tests)
100""" % {"arches": " ".join(ARCHES),
101       "modes": " ".join(MODES),
102       "targets": ", ".join(TARGETS)}
103
104TESTSUITES_TARGETS = {"benchmarks": "d8",
105              "bigint": "bigint_shell",
106              "cctest": "cctest",
107              "debugger": "d8",
108              "fuzzer": "v8_fuzzers",
109              "inspector": "inspector-test",
110              "intl": "d8",
111              "message": "d8",
112              "mjsunit": "d8",
113              "mozilla": "d8",
114              "test262": "d8",
115              "unittests": "unittests",
116              "wasm-api-tests": "wasm_api_tests",
117              "wasm-js": "d8",
118              "wasm-spec-tests": "d8",
119              "webkit": "d8"}
120
121OUTDIR = "out"
122
123def _Which(cmd):
124  for path in os.environ["PATH"].split(os.pathsep):
125    if os.path.exists(os.path.join(path, cmd)):
126      return os.path.join(path, cmd)
127  return None
128
129def DetectGoma():
130  if os.environ.get("GOMA_DIR"):
131    return os.environ.get("GOMA_DIR")
132  if os.environ.get("GOMADIR"):
133    return os.environ.get("GOMADIR")
134  # There is a copy of goma in depot_tools, but it might not be in use on
135  # this machine.
136  goma = _Which("goma_ctl")
137  if goma is None: return None
138  cipd_bin = os.path.join(os.path.dirname(goma), ".cipd_bin")
139  if not os.path.exists(cipd_bin): return None
140  goma_auth = os.path.expanduser("~/.goma_client_oauth2_config")
141  if not os.path.exists(goma_auth): return None
142  return cipd_bin
143
144GOMADIR = DetectGoma()
145IS_GOMA_MACHINE = GOMADIR is not None
146
147USE_GOMA = "true" if IS_GOMA_MACHINE else "false"
148
149RELEASE_ARGS_TEMPLATE = """\
150is_component_build = false
151is_debug = false
152%s
153use_goma = {GOMA}
154v8_enable_backtrace = true
155v8_enable_disassembler = true
156v8_enable_object_print = true
157v8_enable_verify_heap = true
158dcheck_always_on = false
159""".replace("{GOMA}", USE_GOMA)
160
161DEBUG_ARGS_TEMPLATE = """\
162is_component_build = true
163is_debug = true
164symbol_level = 2
165%s
166use_goma = {GOMA}
167v8_enable_backtrace = true
168v8_enable_fast_mksnapshot = true
169v8_enable_slow_dchecks = true
170v8_optimized_debug = false
171""".replace("{GOMA}", USE_GOMA)
172
173OPTDEBUG_ARGS_TEMPLATE = """\
174is_component_build = true
175is_debug = true
176symbol_level = 1
177%s
178use_goma = {GOMA}
179v8_enable_backtrace = true
180v8_enable_fast_mksnapshot = true
181v8_enable_verify_heap = true
182v8_optimized_debug = true
183""".replace("{GOMA}", USE_GOMA)
184
185ARGS_TEMPLATES = {
186  "release": RELEASE_ARGS_TEMPLATE,
187  "debug": DEBUG_ARGS_TEMPLATE,
188  "optdebug": OPTDEBUG_ARGS_TEMPLATE
189}
190
191def PrintHelpAndExit():
192  print(__doc__)
193  print(HELP)
194  sys.exit(0)
195
196def PrintCompletionsAndExit():
197  for a in ARCHES:
198    print("%s" % a)
199    for m in MODES:
200      print("%s" % m)
201      print("%s.%s" % (a, m))
202      for t in TARGETS:
203        print("%s" % t)
204        print("%s.%s.%s" % (a, m, t))
205  sys.exit(0)
206
207def _Call(cmd, silent=False):
208  if not silent: print("# %s" % cmd)
209  return subprocess.call(cmd, shell=True)
210
211def _CallWithOutputNoTerminal(cmd):
212  return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
213
214def _CallWithOutput(cmd):
215  print("# %s" % cmd)
216  # The following trickery is required so that the 'cmd' thinks it's running
217  # in a real terminal, while this script gets to intercept its output.
218  parent, child = pty.openpty()
219  p = subprocess.Popen(cmd, shell=True, stdin=child, stdout=child, stderr=child)
220  os.close(child)
221  output = []
222  try:
223    while True:
224      try:
225        data = os.read(parent, 512).decode('utf-8')
226      except OSError as e:
227        if e.errno != errno.EIO: raise
228        break # EIO means EOF on some systems
229      else:
230        if not data: # EOF
231          break
232        print(data, end="")
233        sys.stdout.flush()
234        output.append(data)
235  finally:
236    os.close(parent)
237    p.wait()
238  return p.returncode, "".join(output)
239
240def _Write(filename, content):
241  print("# echo > %s << EOF\n%sEOF" % (filename, content))
242  with open(filename, "w") as f:
243    f.write(content)
244
245def _Notify(summary, body):
246  if _Which('notify-send') is not None:
247    _Call("notify-send '{}' '{}'".format(summary, body), silent=True)
248  else:
249    print("{} - {}".format(summary, body))
250
251def _GetMachine():
252  return platform.machine()
253
254def GetPath(arch, mode):
255  subdir = "%s.%s" % (arch, mode)
256  return os.path.join(OUTDIR, subdir)
257
258def PrepareMksnapshotCmdline(orig_cmdline, path):
259  result = "gdb --args %s/mksnapshot " % path
260  for w in orig_cmdline.split(" "):
261    if w.startswith("gen/") or w.startswith("snapshot_blob"):
262      result += ("%(path)s%(sep)s%(arg)s " %
263                 {"path": path, "sep": os.sep, "arg": w})
264    else:
265      result += "%s " % w
266  return result
267
268class Config(object):
269  def __init__(self,
270               arch,
271               mode,
272               targets,
273               tests=[],
274               clean=False,
275               testrunner_args=[]):
276    self.arch = arch
277    self.mode = mode
278    self.targets = set(targets)
279    self.tests = set(tests)
280    self.testrunner_args = testrunner_args
281    self.clean = clean
282
283  def Extend(self, targets, tests=[], clean=False):
284    self.targets.update(targets)
285    self.tests.update(tests)
286    self.clean |= clean
287
288  def GetTargetCpu(self):
289    cpu = "x86"
290    if self.arch == "android_arm":
291      cpu = "arm"
292    elif self.arch == "android_arm64" or self.arch == "fuchsia_arm64":
293      cpu = "arm64"
294    elif self.arch == "arm64" and _GetMachine() in ("aarch64", "arm64"):
295      # arm64 build host:
296      cpu = "arm64"
297    elif self.arch == "arm" and _GetMachine() in ("aarch64", "arm64"):
298      cpu = "arm"
299    elif self.arch == "loong64" and _GetMachine() == "loongarch64":
300      cpu = "loong64"
301    elif self.arch == "mips64el" and _GetMachine() == "mips64":
302      cpu = "mips64el"
303    elif "64" in self.arch or self.arch == "s390x":
304      # Native x64 or simulator build.
305      cpu = "x64"
306    return ["target_cpu = \"%s\"" % cpu]
307
308  def GetV8TargetCpu(self):
309    if self.arch == "android_arm":
310      v8_cpu = "arm"
311    elif self.arch == "android_arm64" or self.arch == "fuchsia_arm64":
312      v8_cpu = "arm64"
313    elif self.arch in ("arm", "arm64", "mipsel", "mips64el", "ppc", "ppc64",
314                       "riscv64", "s390", "s390x", "loong64"):
315      v8_cpu = self.arch
316    else:
317      return []
318    return ["v8_target_cpu = \"%s\"" % v8_cpu]
319
320  def GetTargetOS(self):
321    if self.arch in ("android_arm", "android_arm64"):
322      return ["target_os = \"android\""]
323    elif self.arch in ("fuchsia_x64", "fuchsia_arm64"):
324      return ["target_os = \"fuchsia\""]
325    return []
326
327  def GetSpecialCompiler(self):
328    if _GetMachine() in ("aarch64", "mips64", "loongarch64"):
329      # We have no prebuilt Clang for arm64, mips64 or loongarch64 on Linux,
330      # so use the system Clang instead.
331      return ["clang_base_path = \"/usr\"", "clang_use_chrome_plugins = false"]
332    return []
333
334  def GetGnArgs(self):
335    # Use only substring before first '-' as the actual mode
336    mode = re.match("([^-]+)", self.mode).group(1)
337    template = ARGS_TEMPLATES[mode]
338    arch_specific = (self.GetTargetCpu() + self.GetV8TargetCpu() +
339                     self.GetTargetOS() + self.GetSpecialCompiler())
340    return template % "\n".join(arch_specific)
341
342  def Build(self):
343    path = GetPath(self.arch, self.mode)
344    args_gn = os.path.join(path, "args.gn")
345    build_ninja = os.path.join(path, "build.ninja")
346    if not os.path.exists(path):
347      print("# mkdir -p %s" % path)
348      os.makedirs(path)
349    if not os.path.exists(args_gn):
350      _Write(args_gn, self.GetGnArgs())
351    if not os.path.exists(build_ninja):
352      code = _Call("gn gen %s" % path)
353      if code != 0: return code
354    elif self.clean:
355      code = _Call("gn clean %s" % path)
356      if code != 0: return code
357    targets = " ".join(self.targets)
358    # The implementation of mksnapshot failure detection relies on
359    # the "pty" module and GDB presence, so skip it on non-Linux.
360    if not USE_PTY:
361      return _Call("autoninja -C %s %s" % (path, targets))
362
363    return_code, output = _CallWithOutput("autoninja -C %s %s" %
364                                          (path, targets))
365    if return_code != 0 and "FAILED:" in output and "snapshot_blob" in output:
366      csa_trap = re.compile("Specify option( --csa-trap-on-node=[^ ]*)")
367      match = csa_trap.search(output)
368      extra_opt = match.group(1) if match else ""
369      cmdline = re.compile("python3 ../../tools/run.py ./mksnapshot (.*)")
370      orig_cmdline = cmdline.search(output).group(1).strip()
371      cmdline = PrepareMksnapshotCmdline(orig_cmdline, path) + extra_opt
372      _Notify("V8 build requires your attention",
373              "Detected mksnapshot failure, re-running in GDB...")
374      _Call(cmdline)
375    return return_code
376
377  def RunTests(self):
378    # Special handling for "mkgrokdump": if it was built, run it.
379    if (self.arch == "x64" and self.mode == "release" and
380        "mkgrokdump" in self.targets):
381      _Call("%s/mkgrokdump > tools/v8heapconst.py" %
382            GetPath(self.arch, self.mode))
383    if not self.tests: return 0
384    if "ALL" in self.tests:
385      tests = ""
386    else:
387      tests = " ".join(self.tests)
388    return _Call('"%s" ' % sys.executable +
389                 os.path.join("tools", "run-tests.py") +
390                 " --outdir=%s %s %s" % (
391                     GetPath(self.arch, self.mode), tests,
392                     " ".join(self.testrunner_args)))
393
394def GetTestBinary(argstring):
395  for suite in TESTSUITES_TARGETS:
396    if argstring.startswith(suite): return TESTSUITES_TARGETS[suite]
397  return None
398
399class ArgumentParser(object):
400  def __init__(self):
401    self.global_targets = set()
402    self.global_tests = set()
403    self.global_actions = set()
404    self.configs = {}
405    self.testrunner_args = []
406
407  def PopulateConfigs(self, arches, modes, targets, tests, clean):
408    for a in arches:
409      for m in modes:
410        path = GetPath(a, m)
411        if path not in self.configs:
412          self.configs[path] = Config(a, m, targets, tests, clean,
413                  self.testrunner_args)
414        else:
415          self.configs[path].Extend(targets, tests)
416
417  def ProcessGlobalActions(self):
418    have_configs = len(self.configs) > 0
419    for action in self.global_actions:
420      impact = ACTIONS[action]
421      if (have_configs):
422        for c in self.configs:
423          self.configs[c].Extend(**impact)
424      else:
425        self.PopulateConfigs(DEFAULT_ARCHES, DEFAULT_MODES, **impact)
426
427  def ParseArg(self, argstring):
428    if argstring in ("-h", "--help", "help"):
429      PrintHelpAndExit()
430    if argstring == "--print-completions":
431      PrintCompletionsAndExit()
432    arches = []
433    modes = []
434    targets = []
435    actions = []
436    tests = []
437    clean = False
438    # Special handling for "mkgrokdump": build it for x64.release.
439    if argstring == "mkgrokdump":
440      self.PopulateConfigs(["x64"], ["release"], ["mkgrokdump"], [], False)
441      return
442    # Specifying a single unit test looks like "unittests/Foo.Bar", test262
443    # tests have names like "S15.4.4.7_A4_T1", don't split these.
444    if argstring.startswith("unittests/") or argstring.startswith("test262/"):
445      words = [argstring]
446    elif argstring.startswith("--"):
447      # Pass all other flags to test runner.
448      self.testrunner_args.append(argstring)
449      return
450    else:
451      # Assume it's a word like "x64.release" -> split at the dot.
452      words = argstring.split('.')
453    if len(words) == 1:
454      word = words[0]
455      if word in ACTIONS:
456        self.global_actions.add(word)
457        return
458      if word in TARGETS:
459        self.global_targets.add(word)
460        return
461      maybe_target = GetTestBinary(word)
462      if maybe_target is not None:
463        self.global_tests.add(word)
464        self.global_targets.add(maybe_target)
465        return
466    for word in words:
467      if word in ARCHES:
468        arches.append(word)
469      elif word in MODES:
470        modes.append(word)
471      elif word in TARGETS:
472        targets.append(word)
473      elif word in ACTIONS:
474        actions.append(word)
475      elif any(map(lambda x: word.startswith(x + "-"), MODES)):
476        modes.append(word)
477      else:
478        print("Didn't understand: %s" % word)
479        sys.exit(1)
480    # Process actions.
481    for action in actions:
482      impact = ACTIONS[action]
483      targets += impact["targets"]
484      tests += impact["tests"]
485      clean |= impact["clean"]
486    # Fill in defaults for things that weren't specified.
487    arches = arches or DEFAULT_ARCHES
488    modes = modes or DEFAULT_MODES
489    targets = targets or DEFAULT_TARGETS
490    # Produce configs.
491    self.PopulateConfigs(arches, modes, targets, tests, clean)
492
493  def ParseArguments(self, argv):
494    if len(argv) == 0:
495      PrintHelpAndExit()
496    for argstring in argv:
497      self.ParseArg(argstring)
498    self.ProcessGlobalActions()
499    for c in self.configs:
500      self.configs[c].Extend(self.global_targets, self.global_tests)
501    return self.configs
502
503def Main(argv):
504  parser = ArgumentParser()
505  configs = parser.ParseArguments(argv[1:])
506  return_code = 0
507  # If we have Goma but it is not running, start it.
508  if (IS_GOMA_MACHINE and
509      _Call("pgrep -x compiler_proxy > /dev/null", silent=True) != 0):
510    _Call("%s/goma_ctl.py ensure_start" % GOMADIR)
511  for c in configs:
512    return_code += configs[c].Build()
513  if return_code == 0:
514    for c in configs:
515      return_code += configs[c].RunTests()
516  if return_code == 0:
517    _Notify('Done!', 'V8 compilation finished successfully.')
518  else:
519    _Notify('Error!', 'V8 compilation finished with errors.')
520  return return_code
521
522if __name__ == "__main__":
523  sys.exit(Main(sys.argv))
524