• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2020 The Pigweed Authors
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5# use this file except in compliance with the License. You may obtain a copy of
6# the License at
7#
8#     https://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations under
14# the License.
15"""Installs and then runs cipd.
16
17This script installs cipd in ./tools/ (if necessary) and then executes it,
18passing through all arguments.
19
20Must be tested with Python 2 and Python 3.
21"""
22
23from __future__ import print_function
24
25import hashlib
26import os
27import platform
28import ssl
29import subprocess
30import sys
31import base64
32
33try:
34    import httplib  # type: ignore
35except ImportError:
36    import http.client as httplib  # type: ignore[no-redef]
37
38try:
39    import urlparse  # type: ignore
40except ImportError:
41    import urllib.parse as urlparse  # type: ignore[no-redef]
42
43# Generated from the following command. May need to be periodically rerun.
44# $ cipd ls infra/tools/cipd | perl -pe "s[.*/][];s/^/    '/;s/\s*$/',\n/;"
45SUPPORTED_PLATFORMS = (
46    'aix-ppc64',
47    'linux-386',
48    'linux-amd64',
49    'linux-arm64',
50    'linux-armv6l',
51    'linux-mips64',
52    'linux-mips64le',
53    'linux-mipsle',
54    'linux-ppc64',
55    'linux-ppc64le',
56    'linux-s390x',
57    'mac-amd64',
58    'mac-arm64',
59    'windows-386',
60    'windows-amd64',
61)
62
63
64class UnsupportedPlatform(Exception):
65    pass
66
67
68try:
69    SCRIPT_DIR = os.path.dirname(__file__)
70except NameError:  # __file__ not defined.
71    try:
72        SCRIPT_DIR = os.path.join(os.environ['PW_ROOT'], 'pw_env_setup', 'py',
73                                  'pw_env_setup', 'cipd_setup')
74    except KeyError:
75        raise Exception('Environment variable PW_ROOT not set')
76
77VERSION_FILE = os.path.join(SCRIPT_DIR, '.cipd_version')
78DIGESTS_FILE = VERSION_FILE + '.digests'
79
80# Put CIPD client in tools so that users can easily get it in their PATH.
81CIPD_HOST = 'chrome-infra-packages.appspot.com'
82
83try:
84    PW_ROOT = os.environ['PW_ROOT']
85except KeyError:
86    try:
87        with open(os.devnull, 'w') as outs:
88            PW_ROOT = subprocess.check_output(
89                ['git', 'rev-parse', '--show-toplevel'],
90                stderr=outs,
91            ).strip().decode('utf-8')
92    except subprocess.CalledProcessError:
93        PW_ROOT = ''
94
95# Get default install dir from environment since args cannot always be passed
96# through this script (args are passed as-is to cipd).
97if 'CIPD_PY_INSTALL_DIR' in os.environ:
98    DEFAULT_INSTALL_DIR = os.environ['CIPD_PY_INSTALL_DIR']
99elif PW_ROOT:
100    DEFAULT_INSTALL_DIR = os.path.join(PW_ROOT, '.cipd')
101else:
102    DEFAULT_INSTALL_DIR = ''
103
104
105def platform_normalized():
106    """Normalize platform into format expected in CIPD paths."""
107
108    try:
109        os_name = platform.system().lower()
110        return {
111            'linux': 'linux',
112            'mac': 'mac',
113            'darwin': 'mac',
114            'windows': 'windows',
115        }[os_name]
116    except KeyError:
117        raise Exception('unrecognized os: {}'.format(os_name))
118
119
120def arch_normalized():
121    """Normalize arch into format expected in CIPD paths."""
122
123    machine = platform.machine()
124    if machine.startswith(('arm', 'aarch')):
125        machine = machine.replace('aarch', 'arm')
126        if machine == 'arm64':
127            return machine
128        return 'armv6l'
129    if machine.endswith('64'):
130        return 'amd64'
131    if machine.endswith('86'):
132        return '386'
133    raise Exception('unrecognized arch: {}'.format(machine))
134
135
136def platform_arch_normalized():
137    platform_arch = '{}-{}'.format(platform_normalized(), arch_normalized())
138
139    # Support `mac-arm64` through Rosetta until `mac-arm64` binaries are ready
140    if platform_arch == 'mac-arm64':
141        platform_arch = 'mac-amd64'
142
143    return platform_arch
144
145
146def user_agent():
147    """Generate a user-agent based on the project name and current hash."""
148
149    try:
150        rev = subprocess.check_output(
151            ['git', '-C', SCRIPT_DIR, 'rev-parse', 'HEAD']).strip()
152    except subprocess.CalledProcessError:
153        rev = '???'
154
155    if isinstance(rev, bytes):
156        rev = rev.decode()
157
158    return 'pigweed-infra/tools/{}'.format(rev)
159
160
161def actual_hash(path):
162    """Hash the file at path and return it."""
163
164    hasher = hashlib.sha256()
165    with open(path, 'rb') as ins:
166        hasher.update(ins.read())
167    return hasher.hexdigest()
168
169
170def expected_hash():
171    """Pulls expected hash from digests file."""
172
173    expected_plat = platform_arch_normalized()
174
175    with open(DIGESTS_FILE, 'r') as ins:
176        for line in ins:
177            line = line.strip()
178            if line.startswith('#') or not line:
179                continue
180            plat, hashtype, hashval = line.split()
181            if (hashtype == 'sha256' and plat == expected_plat):
182                return hashval
183    raise Exception('platform {} not in {}'.format(expected_plat,
184                                                   DIGESTS_FILE))
185
186
187def https_connect_with_proxy(target_url):
188    """Create HTTPSConnection with proxy support."""
189
190    proxy_env = os.environ.get('HTTPS_PROXY') or os.environ.get('https_proxy')
191    if proxy_env in (None, ''):
192        conn = httplib.HTTPSConnection(target_url)
193        return conn
194
195    url = urlparse.urlparse(proxy_env)
196    conn = httplib.HTTPSConnection(url.hostname, url.port)
197    headers = {}
198    if url.username and url.password:
199        auth = '%s:%s' % (url.username, url.password)
200        py_version = sys.version_info.major
201        if py_version >= 3:
202            headers['Proxy-Authorization'] = 'Basic ' + str(
203                base64.b64encode(auth.encode()).decode())
204        else:
205            headers['Proxy-Authorization'] = 'Basic ' + base64.b64encode(auth)
206    conn.set_tunnel(target_url, 443, headers)
207    return conn
208
209
210def client_bytes():
211    """Pull down the CIPD client and return it as a bytes object.
212
213    Often CIPD_HOST returns a 302 FOUND with a pointer to
214    storage.googleapis.com, so this needs to handle redirects, but it
215    shouldn't require the initial response to be a redirect either.
216    """
217
218    with open(VERSION_FILE, 'r') as ins:
219        version = ins.read().strip()
220
221    try:
222        conn = https_connect_with_proxy(CIPD_HOST)
223    except AttributeError:
224        print('=' * 70)
225        print('''
226It looks like this version of Python does not support SSL. This is common
227when using Homebrew. If using Homebrew please run the following commands.
228If not using Homebrew check how your version of Python was built.
229
230brew install openssl  # Probably already installed, but good to confirm.
231brew uninstall python && brew install python
232'''.strip())
233        print('=' * 70)
234        raise
235
236    full_platform = platform_arch_normalized()
237    if full_platform not in SUPPORTED_PLATFORMS:
238        raise UnsupportedPlatform(full_platform)
239
240    path = '/client?platform={}&version={}'.format(full_platform, version)
241
242    for _ in range(10):
243        try:
244            conn.request('GET', path)
245            res = conn.getresponse()
246            # Have to read the response before making a new request, so make
247            # sure we always read it.
248            content = res.read()
249        except ssl.SSLError:
250            print(
251                '\n'
252                'Bootstrap: SSL error in Python when downloading CIPD client.\n'
253                'If using system Python try\n'
254                '\n'
255                '    sudo pip install certifi\n'
256                '\n'
257                'And if on the system Python on a Mac try\n'
258                '\n'
259                '    /Applications/Python 3.6/Install Certificates.command\n'
260                '\n'
261                'If using Homebrew Python try\n'
262                '\n'
263                '    brew install openssl\n'
264                '    brew uninstall python\n'
265                '    brew install python\n'
266                '\n'
267                "If those don't work, address all the potential issues shown \n"
268                'by the following command.\n'
269                '\n'
270                '    brew doctor\n'
271                '\n'
272                "Otherwise, check that your machine's Python can use SSL, "
273                'testing with the httplib module on Python 2 or http.client on '
274                'Python 3.',
275                file=sys.stderr)
276            raise
277
278        # Found client bytes.
279        if res.status == httplib.OK:  # pylint: disable=no-else-return
280            return content
281
282        # Redirecting to another location.
283        elif res.status == httplib.FOUND:
284            location = res.getheader('location')
285            url = urlparse.urlparse(location)
286            if url.netloc != conn.host:
287                conn = https_connect_with_proxy(url.netloc)
288            path = '{}?{}'.format(url.path, url.query)
289
290        # Some kind of error in this response.
291        else:
292            break
293
294    raise Exception('failed to download client from https://{}{}'.format(
295        CIPD_HOST, path))
296
297
298def bootstrap(client, silent=('PW_ENVSETUP_QUIET' in os.environ)):
299    """Bootstrap cipd client installation."""
300
301    client_dir = os.path.dirname(client)
302    if not os.path.isdir(client_dir):
303        os.makedirs(client_dir)
304
305    if not silent:
306        print('Bootstrapping cipd client for {}'.format(
307            platform_arch_normalized()))
308
309    tmp_path = client + '.tmp'
310    with open(tmp_path, 'wb') as tmp:
311        tmp.write(client_bytes())
312
313    expected = expected_hash()
314    actual = actual_hash(tmp_path)
315
316    if expected != actual:
317        raise Exception('digest of downloaded CIPD client is incorrect, '
318                        'check that digests file is current')
319
320    os.chmod(tmp_path, 0o755)
321    os.rename(tmp_path, client)
322
323
324def selfupdate(client):
325    """Update cipd client."""
326
327    cmd = [
328        client,
329        'selfupdate',
330        '-version-file', VERSION_FILE,
331        '-service-url', 'https://{}'.format(CIPD_HOST),
332    ]  # yapf: disable
333    subprocess.check_call(cmd)
334
335
336def _default_client(install_dir):
337    client = os.path.join(install_dir, 'cipd')
338    if os.name == 'nt':
339        client += '.exe'
340    return client
341
342
343def init(install_dir=DEFAULT_INSTALL_DIR, silent=False, client=None):
344    """Install/update cipd client."""
345
346    if not client:
347        client = _default_client(install_dir)
348
349    os.environ['CIPD_HTTP_USER_AGENT_PREFIX'] = user_agent()
350
351    if not os.path.isfile(client):
352        bootstrap(client, silent)
353
354    try:
355        selfupdate(client)
356    except subprocess.CalledProcessError:
357        print('CIPD selfupdate failed. Bootstrapping then retrying...',
358              file=sys.stderr)
359        bootstrap(client)
360        selfupdate(client)
361
362    return client
363
364
365def main(install_dir=DEFAULT_INSTALL_DIR, silent=False):
366    """Install/update cipd client."""
367
368    client = _default_client(install_dir)
369
370    try:
371        init(install_dir=install_dir, silent=silent, client=client)
372
373    except UnsupportedPlatform:
374        # Don't show help message below for this exception.
375        raise
376
377    except Exception:
378        print('Failed to initialize CIPD. Run '
379              '`CIPD_HTTP_USER_AGENT_PREFIX={user_agent}/manual {client} '
380              "selfupdate -version-file '{version_file}'` "
381              'to diagnose if this is persistent.'.format(
382                  user_agent=user_agent(),
383                  client=client,
384                  version_file=VERSION_FILE,
385              ),
386              file=sys.stderr)
387        raise
388
389    return client
390
391
392if __name__ == '__main__':
393    client_exe = main()
394    subprocess.check_call([client_exe] + sys.argv[1:])
395