• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# coding: utf-8
2from __future__ import unicode_literals, division, absolute_import, print_function
3
4import os
5import subprocess
6import sys
7import shutil
8import re
9import json
10import tarfile
11import zipfile
12
13from . import package_root, build_root, other_packages
14from ._pep425 import _pep425tags, _pep425_implementation
15
16if sys.version_info < (3,):
17    str_cls = unicode  # noqa
18else:
19    str_cls = str
20
21
22def run():
23    """
24    Installs required development dependencies. Uses git to checkout other
25    modularcrypto repos for more accurate coverage data.
26    """
27
28    deps_dir = os.path.join(build_root, 'modularcrypto-deps')
29    if os.path.exists(deps_dir):
30        shutil.rmtree(deps_dir, ignore_errors=True)
31    os.mkdir(deps_dir)
32
33    try:
34        print("Staging ci dependencies")
35        _stage_requirements(deps_dir, os.path.join(package_root, 'requires', 'ci'))
36
37        print("Checking out modularcrypto packages for coverage")
38        for other_package in other_packages:
39            pkg_url = 'https://github.com/wbond/%s.git' % other_package
40            pkg_dir = os.path.join(build_root, other_package)
41            if os.path.exists(pkg_dir):
42                print("%s is already present" % other_package)
43                continue
44            print("Cloning %s" % pkg_url)
45            _execute(['git', 'clone', pkg_url], build_root)
46        print()
47
48    except (Exception):
49        if os.path.exists(deps_dir):
50            shutil.rmtree(deps_dir, ignore_errors=True)
51        raise
52
53    return True
54
55
56def _download(url, dest):
57    """
58    Downloads a URL to a directory
59
60    :param url:
61        The URL to download
62
63    :param dest:
64        The path to the directory to save the file in
65
66    :return:
67        The filesystem path to the saved file
68    """
69
70    print('Downloading %s' % url)
71    filename = os.path.basename(url)
72    dest_path = os.path.join(dest, filename)
73
74    if sys.platform == 'win32':
75        powershell_exe = os.path.join('system32\\WindowsPowerShell\\v1.0\\powershell.exe')
76        code = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12;"
77        code += "(New-Object Net.WebClient).DownloadFile('%s', '%s');" % (url, dest_path)
78        _execute([powershell_exe, '-Command', code], dest, 'Unable to connect to')
79
80    else:
81        _execute(
82            ['curl', '-L', '--silent', '--show-error', '-O', url],
83            dest,
84            'Failed to connect to'
85        )
86
87    return dest_path
88
89
90def _tuple_from_ver(version_string):
91    """
92    :param version_string:
93        A unicode dotted version string
94
95    :return:
96        A tuple of integers
97    """
98
99    return tuple(map(int, version_string.split('.')))
100
101
102def _open_archive(path):
103    """
104    :param path:
105        A unicode string of the filesystem path to the archive
106
107    :return:
108        An archive object
109    """
110
111    if path.endswith('.zip'):
112        return zipfile.ZipFile(path, 'r')
113    return tarfile.open(path, 'r')
114
115
116def _list_archive_members(archive):
117    """
118    :param archive:
119        An archive from _open_archive()
120
121    :return:
122        A list of info objects to be used with _info_name() and _extract_info()
123    """
124
125    if isinstance(archive, zipfile.ZipFile):
126        return archive.infolist()
127    return archive.getmembers()
128
129
130def _archive_single_dir(archive):
131    """
132    Check if all members of the archive are in a single top-level directory
133
134    :param archive:
135        An archive from _open_archive()
136
137    :return:
138        None if not a single top level directory in archive, otherwise a
139        unicode string of the top level directory name
140    """
141
142    common_root = None
143    for info in _list_archive_members(archive):
144        fn = _info_name(info)
145        if fn in set(['.', '/']):
146            continue
147        sep = None
148        if '/' in fn:
149            sep = '/'
150        elif '\\' in fn:
151            sep = '\\'
152        if sep is None:
153            root_dir = fn
154        else:
155            root_dir, _ = fn.split(sep, 1)
156        if common_root is None:
157            common_root = root_dir
158        else:
159            if common_root != root_dir:
160                return None
161    return common_root
162
163
164def _info_name(info):
165    """
166    Returns a normalized file path for an archive info object
167
168    :param info:
169        An info object from _list_archive_members()
170
171    :return:
172        A unicode string with all directory separators normalized to "/"
173    """
174
175    if isinstance(info, zipfile.ZipInfo):
176        return info.filename.replace('\\', '/')
177    return info.name.replace('\\', '/')
178
179
180def _extract_info(archive, info):
181    """
182    Extracts the contents of an archive info object
183
184    ;param archive:
185        An archive from _open_archive()
186
187    :param info:
188        An info object from _list_archive_members()
189
190    :return:
191        None, or a byte string of the file contents
192    """
193
194    if isinstance(archive, zipfile.ZipFile):
195        fn = info.filename
196        is_dir = fn.endswith('/') or fn.endswith('\\')
197        out = archive.read(info)
198        if is_dir and out == b'':
199            return None
200        return out
201
202    info_file = archive.extractfile(info)
203    if info_file:
204        return info_file.read()
205    return None
206
207
208def _extract_package(deps_dir, pkg_path, pkg_dir):
209    """
210    Extract a .whl, .zip, .tar.gz or .tar.bz2 into a package path to
211    use when running CI tasks
212
213    :param deps_dir:
214        A unicode string of the directory the package should be extracted to
215
216    :param pkg_path:
217        A unicode string of the path to the archive
218
219    :param pkg_dir:
220        If running setup.py, change to this dir first - a unicode string
221    """
222
223    if pkg_path.endswith('.exe'):
224        try:
225            zf = None
226            zf = zipfile.ZipFile(pkg_path, 'r')
227            # Exes have a PLATLIB folder containing everything we want
228            for zi in zf.infolist():
229                if not zi.filename.startswith('PLATLIB'):
230                    continue
231                data = _extract_info(zf, zi)
232                if data is not None:
233                    dst_path = os.path.join(deps_dir, zi.filename[8:])
234                    dst_dir = os.path.dirname(dst_path)
235                    if not os.path.exists(dst_dir):
236                        os.makedirs(dst_dir)
237                    with open(dst_path, 'wb') as f:
238                        f.write(data)
239        finally:
240            if zf:
241                zf.close()
242        return
243
244    if pkg_path.endswith('.whl'):
245        try:
246            zf = None
247            zf = zipfile.ZipFile(pkg_path, 'r')
248            # Wheels contain exactly what we need and nothing else
249            zf.extractall(deps_dir)
250        finally:
251            if zf:
252                zf.close()
253        return
254
255    # Source archives may contain a bunch of other things, including mutliple
256    # packages, so we must use setup.py/setuptool to install/extract it
257
258    ar = None
259    staging_dir = os.path.join(deps_dir, '_staging')
260    try:
261        ar = _open_archive(pkg_path)
262
263        common_root = _archive_single_dir(ar)
264
265        members = []
266        for info in _list_archive_members(ar):
267            dst_rel_path = _info_name(info)
268            if common_root is not None:
269                dst_rel_path = dst_rel_path[len(common_root) + 1:]
270            members.append((info, dst_rel_path))
271
272        if not os.path.exists(staging_dir):
273            os.makedirs(staging_dir)
274
275        for info, rel_path in members:
276            info_data = _extract_info(ar, info)
277            # Dirs won't return a file
278            if info_data is not None:
279                dst_path = os.path.join(staging_dir, rel_path)
280                dst_dir = os.path.dirname(dst_path)
281                if not os.path.exists(dst_dir):
282                    os.makedirs(dst_dir)
283                with open(dst_path, 'wb') as f:
284                    f.write(info_data)
285
286        setup_dir = staging_dir
287        if pkg_dir:
288            setup_dir = os.path.join(staging_dir, pkg_dir)
289
290        root = os.path.abspath(os.path.join(deps_dir, '..'))
291        install_lib = os.path.basename(deps_dir)
292
293        _execute(
294            [
295                sys.executable,
296                'setup.py',
297                'install',
298                '--root=%s' % root,
299                '--install-lib=%s' % install_lib,
300                '--no-compile'
301            ],
302            setup_dir
303        )
304
305    finally:
306        if ar:
307            ar.close()
308        if staging_dir:
309            shutil.rmtree(staging_dir)
310
311
312def _stage_requirements(deps_dir, path):
313    """
314    Installs requirements without using Python to download, since
315    different services are limiting to TLS 1.2, and older version of
316    Python do not support that
317
318    :param deps_dir:
319        A unicode path to a temporary diretory to use for downloads
320
321    :param path:
322        A unicode filesystem path to a requirements file
323    """
324
325    valid_tags = _pep425tags()
326
327    exe_suffix = None
328    if sys.platform == 'win32' and _pep425_implementation() == 'cp':
329        win_arch = 'win32' if sys.maxsize == 2147483647 else 'win-amd64'
330        version_info = sys.version_info
331        exe_suffix = '.%s-py%d.%d.exe' % (win_arch, version_info[0], version_info[1])
332
333    packages = _parse_requires(path)
334    for p in packages:
335        pkg = p['pkg']
336        pkg_sub_dir = None
337        if p['type'] == 'url':
338            anchor = None
339            if '#' in pkg:
340                pkg, anchor = pkg.split('#', 1)
341                if '&' in anchor:
342                    parts = anchor.split('&')
343                else:
344                    parts = [anchor]
345                for part in parts:
346                    param, value = part.split('=')
347                    if param == 'subdirectory':
348                        pkg_sub_dir = value
349
350            if pkg.endswith('.zip') or pkg.endswith('.tar.gz') or pkg.endswith('.tar.bz2') or pkg.endswith('.whl'):
351                url = pkg
352            else:
353                raise Exception('Unable to install package from URL that is not an archive')
354        else:
355            pypi_json_url = 'https://pypi.org/pypi/%s/json' % pkg
356            json_dest = _download(pypi_json_url, deps_dir)
357            with open(json_dest, 'rb') as f:
358                pkg_info = json.loads(f.read().decode('utf-8'))
359            if os.path.exists(json_dest):
360                os.remove(json_dest)
361
362            latest = pkg_info['info']['version']
363            if p['type'] == '>=':
364                if _tuple_from_ver(p['ver']) > _tuple_from_ver(latest):
365                    raise Exception('Unable to find version %s of %s, newest is %s' % (p['ver'], pkg, latest))
366                version = latest
367            elif p['type'] == '==':
368                if p['ver'] not in pkg_info['releases']:
369                    raise Exception('Unable to find version %s of %s' % (p['ver'], pkg))
370                version = p['ver']
371            else:
372                version = latest
373
374            wheels = {}
375            whl = None
376            tar_bz2 = None
377            tar_gz = None
378            exe = None
379            for download in pkg_info['releases'][version]:
380                if exe_suffix and download['url'].endswith(exe_suffix):
381                    exe = download['url']
382                if download['url'].endswith('.whl'):
383                    parts = os.path.basename(download['url']).split('-')
384                    tag_impl = parts[-3]
385                    tag_abi = parts[-2]
386                    tag_arch = parts[-1].split('.')[0]
387                    wheels[(tag_impl, tag_abi, tag_arch)] = download['url']
388                if download['url'].endswith('.tar.bz2'):
389                    tar_bz2 = download['url']
390                if download['url'].endswith('.tar.gz'):
391                    tar_gz = download['url']
392
393            # Find the most-specific wheel possible
394            for tag in valid_tags:
395                if tag in wheels:
396                    whl = wheels[tag]
397                    break
398
399            if exe_suffix and exe:
400                url = exe
401            elif whl:
402                url = whl
403            elif tar_bz2:
404                url = tar_bz2
405            elif tar_gz:
406                url = tar_gz
407            else:
408                raise Exception('Unable to find suitable download for %s' % pkg)
409
410        local_path = _download(url, deps_dir)
411
412        _extract_package(deps_dir, local_path, pkg_sub_dir)
413
414        os.remove(local_path)
415
416
417def _parse_requires(path):
418    """
419    Does basic parsing of pip requirements files, to allow for
420    using something other than Python to do actual TLS requests
421
422    :param path:
423        A path to a requirements file
424
425    :return:
426        A list of dict objects containing the keys:
427         - 'type' ('any', 'url', '==', '>=')
428         - 'pkg'
429         - 'ver' (if 'type' == '==' or 'type' == '>=')
430    """
431
432    python_version = '.'.join(map(str_cls, sys.version_info[0:2]))
433    sys_platform = sys.platform
434
435    packages = []
436
437    with open(path, 'rb') as f:
438        contents = f.read().decode('utf-8')
439
440    for line in re.split(r'\r?\n', contents):
441        line = line.strip()
442        if not len(line):
443            continue
444        if re.match(r'^\s*#', line):
445            continue
446        if ';' in line:
447            package, cond = line.split(';', 1)
448            package = package.strip()
449            cond = cond.strip()
450            cond = cond.replace('sys_platform', repr(sys_platform))
451            cond = cond.replace('python_version', repr(python_version))
452            if not eval(cond):
453                continue
454        else:
455            package = line.strip()
456
457        if re.match(r'^\s*-r\s*', package):
458            sub_req_file = re.sub(r'^\s*-r\s*', '', package)
459            sub_req_file = os.path.abspath(os.path.join(os.path.dirname(path), sub_req_file))
460            packages.extend(_parse_requires(sub_req_file))
461            continue
462
463        if re.match(r'https?://', package):
464            packages.append({'type': 'url', 'pkg': package})
465            continue
466
467        if '>=' in package:
468            parts = package.split('>=')
469            package = parts[0].strip()
470            ver = parts[1].strip()
471            packages.append({'type': '>=', 'pkg': package, 'ver': ver})
472            continue
473
474        if '==' in package:
475            parts = package.split('==')
476            package = parts[0].strip()
477            ver = parts[1].strip()
478            packages.append({'type': '==', 'pkg': package, 'ver': ver})
479            continue
480
481        if re.search(r'[^ a-zA-Z0-9\-]', package):
482            raise Exception('Unsupported requirements format version constraint: %s' % package)
483
484        packages.append({'type': 'any', 'pkg': package})
485
486    return packages
487
488
489def _execute(params, cwd, retry=None):
490    """
491    Executes a subprocess
492
493    :param params:
494        A list of the executable and arguments to pass to it
495
496    :param cwd:
497        The working directory to execute the command in
498
499    :param retry:
500        If this string is present in stderr, retry the operation
501
502    :return:
503        A 2-element tuple of (stdout, stderr)
504    """
505
506    proc = subprocess.Popen(
507        params,
508        stdout=subprocess.PIPE,
509        stderr=subprocess.PIPE,
510        cwd=cwd
511    )
512    stdout, stderr = proc.communicate()
513    code = proc.wait()
514    if code != 0:
515        if retry and retry in stderr.decode('utf-8'):
516            return _execute(params, cwd)
517        e = OSError('subprocess exit code for "%s" was %d: %s' % (' '.join(params), code, stderr))
518        e.stdout = stdout
519        e.stderr = stderr
520        raise e
521    return (stdout, stderr)
522