• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2014 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# Modified from go/bootstrap.py in Chromium infrastructure's repository to patch
7# out everything but the core toolchain.
8#
9# https://chromium.googlesource.com/infra/infra/
10
11"""Prepares a local hermetic Go installation.
12
13- Downloads and unpacks the Go toolset in ../golang.
14"""
15
16import contextlib
17import logging
18import os
19import platform
20import shutil
21import stat
22import subprocess
23import sys
24import tarfile
25import tempfile
26import urllib
27import zipfile
28
29# TODO(vadimsh): Migrate to new golang.org/x/ paths once Golang moves to
30# git completely.
31
32LOGGER = logging.getLogger(__name__)
33
34
35# /path/to/util/bot
36ROOT = os.path.dirname(os.path.abspath(__file__))
37
38# Where to install Go toolset to. GOROOT would be <TOOLSET_ROOT>/go.
39TOOLSET_ROOT = os.path.join(os.path.dirname(ROOT), 'golang')
40
41# Default workspace with infra go code.
42WORKSPACE = os.path.join(ROOT, 'go')
43
44# Platform depended suffix for executable files.
45EXE_SFX = '.exe' if sys.platform == 'win32' else ''
46
47# Pinned version of Go toolset to download.
48TOOLSET_VERSION = 'go1.15.3'
49
50# Platform dependent portion of a download URL. See http://golang.org/dl/.
51TOOLSET_VARIANTS = {
52  ('darwin', 'x86-64'): 'darwin-amd64.tar.gz',
53  ('linux2', 'x86-32'): 'linux-386.tar.gz',
54  ('linux2', 'x86-64'): 'linux-amd64.tar.gz',
55  ('win32', 'x86-32'): 'windows-386.zip',
56  ('win32', 'x86-64'): 'windows-amd64.zip',
57}
58
59# Download URL root.
60DOWNLOAD_URL_PREFIX = 'https://storage.googleapis.com/golang'
61
62
63class Failure(Exception):
64  """Bootstrap failed."""
65
66
67def get_toolset_url():
68  """URL of a platform specific Go toolset archive."""
69  # TODO(vadimsh): Support toolset for cross-compilation.
70  arch = {
71    'amd64': 'x86-64',
72    'x86_64': 'x86-64',
73    'i386': 'x86-32',
74    'x86': 'x86-32',
75  }.get(platform.machine().lower())
76  variant = TOOLSET_VARIANTS.get((sys.platform, arch))
77  if not variant:
78    # TODO(vadimsh): Compile go lang from source.
79    raise Failure('Unrecognized platform')
80  return '%s/%s.%s' % (DOWNLOAD_URL_PREFIX, TOOLSET_VERSION, variant)
81
82
83def read_file(path):
84  """Returns contents of a given file or None if not readable."""
85  assert isinstance(path, (list, tuple))
86  try:
87    with open(os.path.join(*path), 'r') as f:
88      return f.read()
89  except IOError:
90    return None
91
92
93def write_file(path, data):
94  """Writes |data| to a file."""
95  assert isinstance(path, (list, tuple))
96  with open(os.path.join(*path), 'w') as f:
97    f.write(data)
98
99
100def remove_directory(path):
101  """Recursively removes a directory."""
102  assert isinstance(path, (list, tuple))
103  p = os.path.join(*path)
104  if not os.path.exists(p):
105    return
106  LOGGER.info('Removing %s', p)
107  # Crutch to remove read-only file (.git/* in particular) on Windows.
108  def onerror(func, path, _exc_info):
109    if not os.access(path, os.W_OK):
110      os.chmod(path, stat.S_IWUSR)
111      func(path)
112    else:
113      raise
114  shutil.rmtree(p, onerror=onerror if sys.platform == 'win32' else None)
115
116
117def install_toolset(toolset_root, url):
118  """Downloads and installs Go toolset.
119
120  GOROOT would be <toolset_root>/go/.
121  """
122  if not os.path.exists(toolset_root):
123    os.makedirs(toolset_root)
124  pkg_path = os.path.join(toolset_root, url[url.rfind('/')+1:])
125
126  LOGGER.info('Downloading %s...', url)
127  download_file(url, pkg_path)
128
129  LOGGER.info('Extracting...')
130  if pkg_path.endswith('.zip'):
131    with zipfile.ZipFile(pkg_path, 'r') as f:
132      f.extractall(toolset_root)
133  elif pkg_path.endswith('.tar.gz'):
134    with tarfile.open(pkg_path, 'r:gz') as f:
135      f.extractall(toolset_root)
136  else:
137    raise Failure('Unrecognized archive format')
138
139  LOGGER.info('Validating...')
140  if not check_hello_world(toolset_root):
141    raise Failure('Something is not right, test program doesn\'t work')
142
143
144def download_file(url, path):
145  """Fetches |url| to |path|."""
146  last_progress = [0]
147  def report(a, b, c):
148    progress = int(a * b * 100.0 / c)
149    if progress != last_progress[0]:
150      print >> sys.stderr, 'Downloading... %d%%' % progress
151      last_progress[0] = progress
152  # TODO(vadimsh): Use something less crippled, something that validates SSL.
153  urllib.urlretrieve(url, path, reporthook=report)
154
155
156@contextlib.contextmanager
157def temp_dir(path):
158  """Creates a temporary directory, then deletes it."""
159  tmp = tempfile.mkdtemp(dir=path)
160  try:
161    yield tmp
162  finally:
163    remove_directory([tmp])
164
165
166def check_hello_world(toolset_root):
167  """Compiles and runs 'hello world' program to verify that toolset works."""
168  with temp_dir(toolset_root) as tmp:
169    path = os.path.join(tmp, 'hello.go')
170    write_file([path], r"""
171        package main
172        func main() { println("hello, world\n") }
173    """)
174    out = subprocess.check_output(
175        [get_go_exe(toolset_root), 'run', path],
176        env=get_go_environ(toolset_root, tmp),
177        stderr=subprocess.STDOUT)
178    if out.strip() != 'hello, world':
179      LOGGER.error('Failed to run sample program:\n%s', out)
180      return False
181    return True
182
183
184def ensure_toolset_installed(toolset_root):
185  """Installs or updates Go toolset if necessary.
186
187  Returns True if new toolset was installed.
188  """
189  installed = read_file([toolset_root, 'INSTALLED_TOOLSET'])
190  available = get_toolset_url()
191  if installed == available:
192    LOGGER.debug('Go toolset is up-to-date: %s', TOOLSET_VERSION)
193    return False
194
195  LOGGER.info('Installing Go toolset.')
196  LOGGER.info('  Old toolset is %s', installed)
197  LOGGER.info('  New toolset is %s', available)
198  remove_directory([toolset_root])
199  install_toolset(toolset_root, available)
200  LOGGER.info('Go toolset installed: %s', TOOLSET_VERSION)
201  write_file([toolset_root, 'INSTALLED_TOOLSET'], available)
202  return True
203
204
205def get_go_environ(
206    toolset_root,
207    workspace=None):
208  """Returns a copy of os.environ with added GO* environment variables.
209
210  Overrides GOROOT, GOPATH and GOBIN. Keeps everything else. Idempotent.
211
212  Args:
213    toolset_root: GOROOT would be <toolset_root>/go.
214    workspace: main workspace directory or None if compiling in GOROOT.
215  """
216  env = os.environ.copy()
217  env['GOROOT'] = os.path.join(toolset_root, 'go')
218  if workspace:
219    env['GOBIN'] = os.path.join(workspace, 'bin')
220  else:
221    env.pop('GOBIN', None)
222
223  all_go_paths = []
224  if workspace:
225    all_go_paths.append(workspace)
226  env['GOPATH'] = os.pathsep.join(all_go_paths)
227
228  # New PATH entries.
229  paths_to_add = [
230    os.path.join(env['GOROOT'], 'bin'),
231    env.get('GOBIN'),
232  ]
233
234  # Make sure not to add duplicates entries to PATH over and over again when
235  # get_go_environ is invoked multiple times.
236  path = env['PATH'].split(os.pathsep)
237  paths_to_add = [p for p in paths_to_add if p and p not in path]
238  env['PATH'] = os.pathsep.join(paths_to_add + path)
239
240  return env
241
242
243def get_go_exe(toolset_root):
244  """Returns path to go executable."""
245  return os.path.join(toolset_root, 'go', 'bin', 'go' + EXE_SFX)
246
247
248def bootstrap(logging_level):
249  """Installs all dependencies in default locations.
250
251  Supposed to be called at the beginning of some script (it modifies logger).
252
253  Args:
254    logging_level: logging level of bootstrap process.
255  """
256  logging.basicConfig()
257  LOGGER.setLevel(logging_level)
258  ensure_toolset_installed(TOOLSET_ROOT)
259
260
261def prepare_go_environ():
262  """Returns dict with environment variables to set to use Go toolset.
263
264  Installs or updates the toolset if necessary.
265  """
266  bootstrap(logging.INFO)
267  return get_go_environ(TOOLSET_ROOT, WORKSPACE)
268
269
270def find_executable(name, workspaces):
271  """Returns full path to an executable in some bin/ (in GOROOT or GOBIN)."""
272  basename = name
273  if EXE_SFX and basename.endswith(EXE_SFX):
274    basename = basename[:-len(EXE_SFX)]
275  roots = [os.path.join(TOOLSET_ROOT, 'go', 'bin')]
276  for path in workspaces:
277    roots.extend([
278      os.path.join(path, 'bin'),
279    ])
280  for root in roots:
281    full_path = os.path.join(root, basename + EXE_SFX)
282    if os.path.exists(full_path):
283      return full_path
284  return name
285
286
287def main(args):
288  if args:
289    print >> sys.stderr, sys.modules[__name__].__doc__,
290    return 2
291  bootstrap(logging.DEBUG)
292  return 0
293
294
295if __name__ == '__main__':
296  sys.exit(main(sys.argv[1:]))
297