• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2020 The Pigweed Authors
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5# use this file except in compliance with the License. You may obtain a copy of
6# the License at
7#
8#     https://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations under
14# the License.
15"""Installs or updates prebuilt tools.
16
17Must be tested with Python 2 and Python 3.
18
19The stdout of this script is meant to be executed by the invoking shell.
20"""
21
22from __future__ import print_function
23
24import collections
25import hashlib
26import json
27import os
28import platform as platform_module
29import re
30import subprocess
31import sys
32
33
34def check_auth(cipd, package_files, cipd_service_account, spin):
35    """Check have access to CIPD pigweed directory."""
36    cmd = [cipd]
37    extra_args = []
38    if cipd_service_account:
39        extra_args.extend(['-service-account-json', cipd_service_account])
40
41    paths = []
42    for package_file in package_files:
43        with open(package_file, 'r') as ins:
44            # This is an expensive RPC, so only check the first few entries
45            # in each file.
46            for i, entry in enumerate(json.load(ins).get('packages', ())):
47                if i >= 3:
48                    break
49                parts = entry['path'].split('/')
50                while '${' in parts[-1]:
51                    parts.pop(-1)
52                paths.append('/'.join(parts))
53
54    username = None
55    try:
56        output = subprocess.check_output(
57            cmd + ['auth-info'] + extra_args, stderr=subprocess.STDOUT
58        ).decode()
59        logged_in = True
60
61        match = re.search(r'Logged in as (\S*)\.', output)
62        if match:
63            username = match.group(1)
64
65    except subprocess.CalledProcessError:
66        logged_in = False
67
68    def _check_all_paths():
69        inaccessible_paths = []
70
71        for path in paths:
72            # Not catching CalledProcessError because 'cipd ls' seems to never
73            # return an error code unless it can't reach the CIPD server.
74            output = subprocess.check_output(
75                cmd + ['ls', path] + extra_args, stderr=subprocess.STDOUT
76            ).decode()
77            if 'No matching packages' not in output:
78                continue
79
80            # 'cipd ls' only lists sub-packages but ignores any packages at the
81            # given path. 'cipd instances' will give versions of that package.
82            # 'cipd instances' does use an error code if there's no such package
83            # or that package is inaccessible.
84            try:
85                subprocess.check_output(
86                    cmd + ['instances', path] + extra_args,
87                    stderr=subprocess.STDOUT,
88                )
89            except subprocess.CalledProcessError:
90                inaccessible_paths.append(path)
91
92        return inaccessible_paths
93
94    inaccessible_paths = _check_all_paths()
95
96    if inaccessible_paths and not logged_in:
97        with spin.pause():
98            stderr = lambda *args: print(*args, file=sys.stderr)
99            stderr()
100            stderr(
101                'Not logged in to CIPD and no anonymous access to the '
102                'following CIPD paths:'
103            )
104            for path in inaccessible_paths:
105                stderr('  {}'.format(path))
106            stderr()
107            stderr('Attempting CIPD login')
108            try:
109                # Note that with -service-account-json, auth-login is a no-op.
110                subprocess.check_call(cmd + ['auth-login'] + extra_args)
111            except subprocess.CalledProcessError:
112                stderr('CIPD login failed')
113                return False
114
115        inaccessible_paths = _check_all_paths()
116
117    if inaccessible_paths:
118        stderr = lambda *args: print(*args, file=sys.stderr)
119        stderr('=' * 60)
120        username_part = ''
121        if username:
122            username_part = '({}) '.format(username)
123        stderr(
124            'Your account {}does not have access to the following '
125            'paths'.format(username_part)
126        )
127        stderr('(or they do not exist)')
128        for path in inaccessible_paths:
129            stderr('  {}'.format(path))
130        stderr('=' * 60)
131        return False
132
133    return True
134
135
136def platform(rosetta=False):
137    """Return the CIPD platform string of the current system."""
138    osname = {
139        'darwin': 'mac',
140        'linux': 'linux',
141        'windows': 'windows',
142    }[platform_module.system().lower()]
143
144    if platform_module.machine().startswith(('aarch64', 'armv8')):
145        arch = 'arm64'
146    elif platform_module.machine() == 'x86_64':
147        arch = 'amd64'
148    elif platform_module.machine() == 'i686':
149        arch = 'i386'
150    else:
151        arch = platform_module.machine()
152
153    platform_arch = '{}-{}'.format(osname, arch).lower()
154
155    # Support `mac-arm64` through Rosetta until `mac-arm64` binaries are ready
156    if platform_arch == 'mac-arm64' and rosetta:
157        return 'mac-amd64'
158
159    return platform_arch
160
161
162def all_package_files(env_vars, package_files):
163    """Recursively retrieve all package files."""
164
165    result = []
166    to_process = []
167    for pkg_file in package_files:
168        args = []
169        if env_vars:
170            args.append(env_vars.get('PW_PROJECT_ROOT'))
171        args.append(pkg_file)
172
173        # The signature here is os.path.join(a, *p). Pylint doesn't like when
174        # we call os.path.join(*args), but is happy if we instead call
175        # os.path.join(args[0], *args[1:]). Disabling the option on this line
176        # seems to be a less confusing choice.
177        path = os.path.join(*args)  # pylint: disable=no-value-for-parameter
178
179        to_process.append(path)
180
181    while to_process:
182        package_file = to_process.pop(0)
183        result.append(package_file)
184
185        with open(package_file, 'r') as ins:
186            entries = json.load(ins).get('included_files', ())
187
188        for entry in entries:
189            entry = os.path.join(os.path.dirname(package_file), entry)
190
191            if entry not in result and entry not in to_process:
192                to_process.append(entry)
193
194    return result
195
196
197def all_packages(package_files):
198    packages = []
199    for package_file in package_files:
200        name = package_file_name(package_file)
201        with open(package_file, 'r') as ins:
202            file_packages = json.load(ins).get('packages', ())
203            for package in file_packages:
204                if 'subdir' in package:
205                    package['subdir'] = os.path.join(name, package['subdir'])
206                else:
207                    package['subdir'] = name
208            packages.extend(file_packages)
209    return packages
210
211
212def deduplicate_packages(packages):
213    deduped = collections.OrderedDict()
214    for package in reversed(packages):
215        if package['path'] in deduped:
216            del deduped[package['path']]
217        deduped[package['path']] = package
218    return reversed(list(deduped.values()))
219
220
221def write_ensure_file(
222    package_files, ensure_file, platform
223):  # pylint: disable=redefined-outer-name
224    packages = all_packages(package_files)
225    deduped_packages = deduplicate_packages(packages)
226
227    with open(ensure_file, 'w') as outs:
228        outs.write(
229            '$VerifiedPlatform linux-amd64\n'
230            '$VerifiedPlatform mac-amd64\n'
231            '$ParanoidMode CheckPresence\n'
232        )
233
234        for pkg in deduped_packages:
235            # If this is a new-style package manifest platform handling must
236            # be done here instead of by the cipd executable.
237            if 'platforms' in pkg and platform not in pkg['platforms']:
238                continue
239
240            outs.write('@Subdir {}\n'.format(pkg.get('subdir', '')))
241            outs.write('{} {}\n'.format(pkg['path'], ' '.join(pkg['tags'])))
242
243
244def package_file_name(package_file):
245    return os.path.basename(os.path.splitext(package_file)[0])
246
247
248def package_installation_path(root_install_dir, package_file):
249    """Returns the package installation path.
250
251    Args:
252      root_install_dir: The CIPD installation directory.
253      package_file: The path to the .json package definition file.
254    """
255    return os.path.join(
256        root_install_dir, 'packages', package_file_name(package_file)
257    )
258
259
260def update(  # pylint: disable=too-many-locals
261    cipd,
262    package_files,
263    root_install_dir,
264    cache_dir,
265    rosetta=False,
266    env_vars=None,
267    spin=None,
268    trust_hash=False,
269):
270    """Grab the tools listed in ensure_files."""
271
272    package_files = all_package_files(env_vars, package_files)
273
274    # TODO(mohrr) use os.makedirs(..., exist_ok=True).
275    if not os.path.isdir(root_install_dir):
276        os.makedirs(root_install_dir)
277
278    # This file is read by 'pw doctor' which needs to know which package files
279    # were used in the environment.
280    package_files_file = os.path.join(
281        root_install_dir, '_all_package_files.json'
282    )
283    with open(package_files_file, 'w') as outs:
284        json.dump(package_files, outs, indent=2)
285
286    if env_vars:
287        env_vars.prepend('PATH', root_install_dir)
288        env_vars.set('PW_CIPD_INSTALL_DIR', root_install_dir)
289        env_vars.set('CIPD_CACHE_DIR', cache_dir)
290
291    pw_root = None
292
293    if env_vars:
294        pw_root = env_vars.get('PW_ROOT', None)
295    if not pw_root:
296        pw_root = os.environ['PW_ROOT']
297
298    plat = platform(rosetta)
299
300    ensure_file = os.path.join(root_install_dir, 'packages.ensure')
301    write_ensure_file(package_files, ensure_file, plat)
302
303    install_dir = os.path.join(root_install_dir, 'packages')
304
305    cmd = [
306        cipd,
307        'ensure',
308        '-ensure-file',
309        ensure_file,
310        '-root',
311        install_dir,
312        '-log-level',
313        'debug',
314        '-json-output',
315        os.path.join(root_install_dir, 'packages.json'),
316        '-cache-dir',
317        cache_dir,
318        '-max-threads',
319        '0',  # 0 means use CPU count.
320    ]
321
322    cipd_service_account = None
323    if env_vars:
324        cipd_service_account = env_vars.get('PW_CIPD_SERVICE_ACCOUNT_JSON')
325    if not cipd_service_account:
326        cipd_service_account = os.environ.get('PW_CIPD_SERVICE_ACCOUNT_JSON')
327    if cipd_service_account:
328        cmd.extend(['-service-account-json', cipd_service_account])
329
330    hasher = hashlib.sha256()
331    encoded = '\0'.join(cmd)
332    if hasattr(encoded, 'encode'):
333        encoded = encoded.encode()
334    hasher.update(encoded)
335    with open(ensure_file, 'rb') as ins:
336        hasher.update(ins.read())
337    digest = hasher.hexdigest()
338
339    with open(os.path.join(root_install_dir, 'hash.log'), 'w') as hashlog:
340        print('calculated digest:', digest, file=hashlog)
341
342        hash_file = os.path.join(root_install_dir, 'packages.sha256')
343        print('hash file path:', hash_file, file=hashlog)
344        print('exists:', os.path.isfile(hash_file), file=hashlog)
345        print('trust_hash:', trust_hash, file=hashlog)
346        if trust_hash and os.path.isfile(hash_file):
347            with open(hash_file, 'r') as ins:
348                digest_file = ins.read().strip()
349                print('contents:', digest_file, file=hashlog)
350                print('equal:', digest == digest_file, file=hashlog)
351                if digest == digest_file:
352                    return True
353
354    if not check_auth(cipd, package_files, cipd_service_account, spin):
355        return False
356
357    log = os.path.join(root_install_dir, 'packages.log')
358    try:
359        with open(log, 'w') as outs:
360            print(*cmd, file=outs)
361            subprocess.check_call(cmd, stdout=outs, stderr=subprocess.STDOUT)
362    except subprocess.CalledProcessError:
363        with open(log, 'r') as ins:
364            sys.stderr.write(ins.read())
365            raise
366
367    with open(hash_file, 'w') as outs:
368        print(digest, file=outs)
369
370    # Set environment variables so tools can later find things under, for
371    # example, 'share'.
372    if env_vars:
373        for package_file in reversed(package_files):
374            name = package_file_name(package_file)
375            file_install_dir = os.path.join(install_dir, name)
376            # Some executables get installed at top-level and some get
377            # installed under 'bin'. A small number of old packages prefix the
378            # entire tree with the platform (e.g., chromium/third_party/tcl).
379            for bin_dir in (
380                file_install_dir,
381                os.path.join(file_install_dir, 'bin'),
382                os.path.join(file_install_dir, plat, 'bin'),
383            ):
384                if os.path.isdir(bin_dir):
385                    env_vars.prepend('PATH', bin_dir)
386            env_vars.set(
387                'PW_{}_CIPD_INSTALL_DIR'.format(name.upper()), file_install_dir
388            )
389
390            # Windows has its own special toolchain.
391            if os.name == 'nt':
392                env_vars.prepend(
393                    'PATH', os.path.join(file_install_dir, 'mingw64', 'bin')
394                )
395
396    return True
397