1# coding: utf-8 2from __future__ import unicode_literals, division, absolute_import, print_function 3 4import cgi 5import codecs 6import coverage 7import imp 8import json 9import os 10import unittest 11import re 12import sys 13import tempfile 14import time 15import platform as _plat 16import subprocess 17from fnmatch import fnmatch 18 19from . import package_name, package_root, other_packages 20 21if sys.version_info < (3,): 22 str_cls = unicode # noqa 23 from urllib2 import URLError 24 from urllib import urlencode 25 from io import open 26else: 27 str_cls = str 28 from urllib.error import URLError 29 from urllib.parse import urlencode 30 31if sys.version_info < (3, 7): 32 Pattern = re._pattern_type 33else: 34 Pattern = re.Pattern 35 36 37def run(ci=False): 38 """ 39 Runs the tests while measuring coverage 40 41 :param ci: 42 If coverage is being run in a CI environment - this triggers trying to 43 run the tests for the rest of modularcrypto and uploading coverage data 44 45 :return: 46 A bool - if the tests ran successfully 47 """ 48 49 xml_report_path = os.path.join(package_root, 'coverage.xml') 50 if os.path.exists(xml_report_path): 51 os.unlink(xml_report_path) 52 53 cov = coverage.Coverage(include='%s/*.py' % package_name) 54 cov.start() 55 56 from .tests import run as run_tests 57 result = run_tests(ci=ci) 58 print() 59 60 if ci: 61 suite = unittest.TestSuite() 62 loader = unittest.TestLoader() 63 for other_package in other_packages: 64 for test_class in _load_package_tests(other_package): 65 suite.addTest(loader.loadTestsFromTestCase(test_class)) 66 67 if suite.countTestCases() > 0: 68 print('Running tests from other modularcrypto packages') 69 sys.stdout.flush() 70 runner_result = unittest.TextTestRunner(stream=sys.stdout, verbosity=1).run(suite) 71 result = runner_result.wasSuccessful() and result 72 print() 73 sys.stdout.flush() 74 75 cov.stop() 76 cov.save() 77 78 cov.report(show_missing=False) 79 print() 80 sys.stdout.flush() 81 if ci: 82 cov.xml_report() 83 84 if ci and result and os.path.exists(xml_report_path): 85 _codecov_submit() 86 print() 87 88 return result 89 90 91def _load_package_tests(name): 92 """ 93 Load the test classes from another modularcrypto package 94 95 :param name: 96 A unicode string of the other package name 97 98 :return: 99 A list of unittest.TestCase classes of the tests for the package 100 """ 101 102 package_dir = os.path.join('..', name) 103 if not os.path.exists(package_dir): 104 return [] 105 106 tests_module_info = imp.find_module('tests', [package_dir]) 107 tests_module = imp.load_module('%s.tests' % name, *tests_module_info) 108 return tests_module.test_classes() 109 110 111def _env_info(): 112 """ 113 :return: 114 A two-element tuple of unicode strings. The first is the name of the 115 environment, the second the root of the repo. The environment name 116 will be one of: "ci-travis", "ci-circle", "ci-appveyor", 117 "ci-github-actions", "local" 118 """ 119 120 if os.getenv('CI') == 'true' and os.getenv('TRAVIS') == 'true': 121 return ('ci-travis', os.getenv('TRAVIS_BUILD_DIR')) 122 123 if os.getenv('CI') == 'True' and os.getenv('APPVEYOR') == 'True': 124 return ('ci-appveyor', os.getenv('APPVEYOR_BUILD_FOLDER')) 125 126 if os.getenv('CI') == 'true' and os.getenv('CIRCLECI') == 'true': 127 return ('ci-circle', os.getcwdu() if sys.version_info < (3,) else os.getcwd()) 128 129 if os.getenv('GITHUB_ACTIONS') == 'true': 130 return ('ci-github-actions', os.getenv('GITHUB_WORKSPACE')) 131 132 return ('local', package_root) 133 134 135def _codecov_submit(): 136 env_name, root = _env_info() 137 138 try: 139 with open(os.path.join(root, 'codecov.json'), 'rb') as f: 140 json_data = json.loads(f.read().decode('utf-8')) 141 except (OSError, ValueError, UnicodeDecodeError, KeyError): 142 print('error reading codecov.json') 143 return 144 145 if json_data.get('disabled'): 146 return 147 148 if env_name == 'ci-travis': 149 # http://docs.travis-ci.com/user/environment-variables/#Default-Environment-Variables 150 build_url = 'https://travis-ci.org/%s/jobs/%s' % (os.getenv('TRAVIS_REPO_SLUG'), os.getenv('TRAVIS_JOB_ID')) 151 query = { 152 'service': 'travis', 153 'branch': os.getenv('TRAVIS_BRANCH'), 154 'build': os.getenv('TRAVIS_JOB_NUMBER'), 155 'pr': os.getenv('TRAVIS_PULL_REQUEST'), 156 'job': os.getenv('TRAVIS_JOB_ID'), 157 'tag': os.getenv('TRAVIS_TAG'), 158 'slug': os.getenv('TRAVIS_REPO_SLUG'), 159 'commit': os.getenv('TRAVIS_COMMIT'), 160 'build_url': build_url, 161 } 162 163 elif env_name == 'ci-appveyor': 164 # http://www.appveyor.com/docs/environment-variables 165 build_url = 'https://ci.appveyor.com/project/%s/build/%s' % ( 166 os.getenv('APPVEYOR_REPO_NAME'), 167 os.getenv('APPVEYOR_BUILD_VERSION') 168 ) 169 query = { 170 'service': "appveyor", 171 'branch': os.getenv('APPVEYOR_REPO_BRANCH'), 172 'build': os.getenv('APPVEYOR_JOB_ID'), 173 'pr': os.getenv('APPVEYOR_PULL_REQUEST_NUMBER'), 174 'job': '/'.join(( 175 os.getenv('APPVEYOR_ACCOUNT_NAME'), 176 os.getenv('APPVEYOR_PROJECT_SLUG'), 177 os.getenv('APPVEYOR_BUILD_VERSION') 178 )), 179 'tag': os.getenv('APPVEYOR_REPO_TAG_NAME'), 180 'slug': os.getenv('APPVEYOR_REPO_NAME'), 181 'commit': os.getenv('APPVEYOR_REPO_COMMIT'), 182 'build_url': build_url, 183 } 184 185 elif env_name == 'ci-circle': 186 # https://circleci.com/docs/environment-variables 187 query = { 188 'service': 'circleci', 189 'branch': os.getenv('CIRCLE_BRANCH'), 190 'build': os.getenv('CIRCLE_BUILD_NUM'), 191 'pr': os.getenv('CIRCLE_PR_NUMBER'), 192 'job': os.getenv('CIRCLE_BUILD_NUM') + "." + os.getenv('CIRCLE_NODE_INDEX'), 193 'tag': os.getenv('CIRCLE_TAG'), 194 'slug': os.getenv('CIRCLE_PROJECT_USERNAME') + "/" + os.getenv('CIRCLE_PROJECT_REPONAME'), 195 'commit': os.getenv('CIRCLE_SHA1'), 196 'build_url': os.getenv('CIRCLE_BUILD_URL'), 197 } 198 199 elif env_name == 'ci-github-actions': 200 branch = '' 201 tag = '' 202 ref = os.getenv('GITHUB_REF', '') 203 if ref.startswith('refs/tags/'): 204 tag = ref[10:] 205 elif ref.startswith('refs/heads/'): 206 branch = ref[11:] 207 208 impl = _plat.python_implementation() 209 major, minor = _plat.python_version_tuple()[0:2] 210 build_name = '%s %s %s.%s' % (_platform_name(), impl, major, minor) 211 212 query = { 213 'service': 'custom', 214 'token': json_data['token'], 215 'branch': branch, 216 'tag': tag, 217 'slug': os.getenv('GITHUB_REPOSITORY'), 218 'commit': os.getenv('GITHUB_SHA'), 219 'build_url': 'https://github.com/wbond/oscrypto/commit/%s/checks' % os.getenv('GITHUB_SHA'), 220 'name': 'GitHub Actions %s on %s' % (build_name, os.getenv('RUNNER_OS')) 221 } 222 223 else: 224 if not os.path.exists(os.path.join(root, '.git')): 225 print('git repository not found, not submitting coverage data') 226 return 227 git_status = _git_command(['status', '--porcelain'], root) 228 if git_status != '': 229 print('git repository has uncommitted changes, not submitting coverage data') 230 return 231 232 branch = _git_command(['rev-parse', '--abbrev-ref', 'HEAD'], root) 233 commit = _git_command(['rev-parse', '--verify', 'HEAD'], root) 234 tag = _git_command(['name-rev', '--tags', '--name-only', commit], root) 235 impl = _plat.python_implementation() 236 major, minor = _plat.python_version_tuple()[0:2] 237 build_name = '%s %s %s.%s' % (_platform_name(), impl, major, minor) 238 query = { 239 'branch': branch, 240 'commit': commit, 241 'slug': json_data['slug'], 242 'token': json_data['token'], 243 'build': build_name, 244 } 245 if tag != 'undefined': 246 query['tag'] = tag 247 248 payload = 'PLATFORM=%s\n' % _platform_name() 249 payload += 'PYTHON_VERSION=%s %s\n' % (_plat.python_version(), _plat.python_implementation()) 250 if 'oscrypto' in sys.modules: 251 payload += 'OSCRYPTO_BACKEND=%s\n' % sys.modules['oscrypto'].backend() 252 payload += '<<<<<< ENV\n' 253 254 for path in _list_files(root): 255 payload += path + '\n' 256 payload += '<<<<<< network\n' 257 258 payload += '# path=coverage.xml\n' 259 with open(os.path.join(root, 'coverage.xml'), 'r', encoding='utf-8') as f: 260 payload += f.read() + '\n' 261 payload += '<<<<<< EOF\n' 262 263 url = 'https://codecov.io/upload/v4' 264 headers = { 265 'Accept': 'text/plain' 266 } 267 filtered_query = {} 268 for key in query: 269 value = query[key] 270 if value == '' or value is None: 271 continue 272 filtered_query[key] = value 273 274 print('Submitting coverage info to codecov.io') 275 info = _do_request( 276 'POST', 277 url, 278 headers, 279 query_params=filtered_query 280 ) 281 282 encoding = info[1] or 'utf-8' 283 text = info[2].decode(encoding).strip() 284 parts = text.split() 285 upload_url = parts[1] 286 287 headers = { 288 'Content-Type': 'text/plain', 289 'x-amz-acl': 'public-read', 290 'x-amz-storage-class': 'REDUCED_REDUNDANCY' 291 } 292 293 print('Uploading coverage data to codecov.io S3 bucket') 294 _do_request( 295 'PUT', 296 upload_url, 297 headers, 298 data=payload.encode('utf-8') 299 ) 300 301 302def _git_command(params, cwd): 303 """ 304 Executes a git command, returning the output 305 306 :param params: 307 A list of the parameters to pass to git 308 309 :param cwd: 310 The working directory to execute git in 311 312 :return: 313 A 2-element tuple of (stdout, stderr) 314 """ 315 316 proc = subprocess.Popen( 317 ['git'] + params, 318 stdout=subprocess.PIPE, 319 stderr=subprocess.STDOUT, 320 cwd=cwd 321 ) 322 stdout, stderr = proc.communicate() 323 code = proc.wait() 324 if code != 0: 325 e = OSError('git exit code was non-zero') 326 e.stdout = stdout 327 raise e 328 return stdout.decode('utf-8').strip() 329 330 331def _parse_env_var_file(data): 332 """ 333 Parses a basic VAR="value data" file contents into a dict 334 335 :param data: 336 A unicode string of the file data 337 338 :return: 339 A dict of parsed name/value data 340 """ 341 342 output = {} 343 for line in data.splitlines(): 344 line = line.strip() 345 if not line or '=' not in line: 346 continue 347 parts = line.split('=') 348 if len(parts) != 2: 349 continue 350 name = parts[0] 351 value = parts[1] 352 if len(value) > 1: 353 if value[0] == '"' and value[-1] == '"': 354 value = value[1:-1] 355 output[name] = value 356 return output 357 358 359def _platform_name(): 360 """ 361 Returns information about the current operating system and version 362 363 :return: 364 A unicode string containing the OS name and version 365 """ 366 367 if sys.platform == 'darwin': 368 version = _plat.mac_ver()[0] 369 _plat_ver_info = tuple(map(int, version.split('.'))) 370 if _plat_ver_info < (10, 12): 371 name = 'OS X' 372 else: 373 name = 'macOS' 374 return '%s %s' % (name, version) 375 376 elif sys.platform == 'win32': 377 _win_ver = sys.getwindowsversion() 378 _plat_ver_info = (_win_ver[0], _win_ver[1]) 379 return 'Windows %s' % _plat.win32_ver()[0] 380 381 elif sys.platform in ['linux', 'linux2']: 382 if os.path.exists('/etc/os-release'): 383 with open('/etc/os-release', 'r', encoding='utf-8') as f: 384 pairs = _parse_env_var_file(f.read()) 385 if 'NAME' in pairs and 'VERSION_ID' in pairs: 386 return '%s %s' % (pairs['NAME'], pairs['VERSION_ID']) 387 version = pairs['VERSION_ID'] 388 elif 'PRETTY_NAME' in pairs: 389 return pairs['PRETTY_NAME'] 390 elif 'NAME' in pairs: 391 return pairs['NAME'] 392 else: 393 raise ValueError('No suitable version info found in /etc/os-release') 394 elif os.path.exists('/etc/lsb-release'): 395 with open('/etc/lsb-release', 'r', encoding='utf-8') as f: 396 pairs = _parse_env_var_file(f.read()) 397 if 'DISTRIB_DESCRIPTION' in pairs: 398 return pairs['DISTRIB_DESCRIPTION'] 399 else: 400 raise ValueError('No suitable version info found in /etc/lsb-release') 401 else: 402 return 'Linux' 403 404 else: 405 return '%s %s' % (_plat.system(), _plat.release()) 406 407 408def _list_files(root): 409 """ 410 Lists all of the files in a directory, taking into account any .gitignore 411 file that is present 412 413 :param root: 414 A unicode filesystem path 415 416 :return: 417 A list of unicode strings, containing paths of all files not ignored 418 by .gitignore with root, using relative paths 419 """ 420 421 dir_patterns, file_patterns = _gitignore(root) 422 paths = [] 423 prefix = os.path.abspath(root) + os.sep 424 for base, dirs, files in os.walk(root): 425 for d in dirs: 426 for dir_pattern in dir_patterns: 427 if fnmatch(d, dir_pattern): 428 dirs.remove(d) 429 break 430 for f in files: 431 skip = False 432 for file_pattern in file_patterns: 433 if fnmatch(f, file_pattern): 434 skip = True 435 break 436 if skip: 437 continue 438 full_path = os.path.join(base, f) 439 if full_path[:len(prefix)] == prefix: 440 full_path = full_path[len(prefix):] 441 paths.append(full_path) 442 return sorted(paths) 443 444 445def _gitignore(root): 446 """ 447 Parses a .gitignore file and returns patterns to match dirs and files. 448 Only basic gitignore patterns are supported. Pattern negation, ** wildcards 449 and anchored patterns are not currently implemented. 450 451 :param root: 452 A unicode string of the path to the git repository 453 454 :return: 455 A 2-element tuple: 456 - 0: a list of unicode strings to match against dirs 457 - 1: a list of unicode strings to match against dirs and files 458 """ 459 460 gitignore_path = os.path.join(root, '.gitignore') 461 462 dir_patterns = ['.git'] 463 file_patterns = [] 464 465 if not os.path.exists(gitignore_path): 466 return (dir_patterns, file_patterns) 467 468 with open(gitignore_path, 'r', encoding='utf-8') as f: 469 for line in f.readlines(): 470 line = line.strip() 471 if not line: 472 continue 473 if line.startswith('#'): 474 continue 475 if '**' in line: 476 raise NotImplementedError('gitignore ** wildcards are not implemented') 477 if line.startswith('!'): 478 raise NotImplementedError('gitignore pattern negation is not implemented') 479 if line.startswith('/'): 480 raise NotImplementedError('gitignore anchored patterns are not implemented') 481 if line.startswith('\\#'): 482 line = '#' + line[2:] 483 if line.startswith('\\!'): 484 line = '!' + line[2:] 485 if line.endswith('/'): 486 dir_patterns.append(line[:-1]) 487 else: 488 file_patterns.append(line) 489 490 return (dir_patterns, file_patterns) 491 492 493def _do_request(method, url, headers, data=None, query_params=None, timeout=20): 494 """ 495 Performs an HTTP request 496 497 :param method: 498 A unicode string of 'POST' or 'PUT' 499 500 :param url; 501 A unicode string of the URL to request 502 503 :param headers: 504 A dict of unicode strings, where keys are header names and values are 505 the header values. 506 507 :param data: 508 A dict of unicode strings (to be encoded as 509 application/x-www-form-urlencoded), or a byte string of data. 510 511 :param query_params: 512 A dict of unicode keys and values to pass as query params 513 514 :param timeout: 515 An integer number of seconds to use as the timeout 516 517 :return: 518 A 3-element tuple: 519 - 0: A unicode string of the response content-type 520 - 1: A unicode string of the response encoding, or None 521 - 2: A byte string of the response body 522 """ 523 524 if query_params: 525 url += '?' + urlencode(query_params).replace('+', '%20') 526 527 if isinstance(data, dict): 528 data_bytes = {} 529 for key in data: 530 data_bytes[key.encode('utf-8')] = data[key].encode('utf-8') 531 data = urlencode(data_bytes) 532 headers['Content-Type'] = 'application/x-www-form-urlencoded' 533 if isinstance(data, str_cls): 534 raise TypeError('data must be a byte string') 535 536 try: 537 tempfd, tempf_path = tempfile.mkstemp('-coverage') 538 os.write(tempfd, data or b'') 539 os.close(tempfd) 540 541 if sys.platform == 'win32': 542 powershell_exe = os.path.join('system32\\WindowsPowerShell\\v1.0\\powershell.exe') 543 code = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12;" 544 code += "$wc = New-Object Net.WebClient;" 545 for key in headers: 546 code += "$wc.Headers.add('%s','%s');" % (key, headers[key]) 547 code += "$out = $wc.UploadFile('%s', '%s', '%s');" % (url, method, tempf_path) 548 code += "[System.Text.Encoding]::GetEncoding('ISO-8859-1').GetString($wc.ResponseHeaders.ToByteArray())" 549 550 # To properly obtain bytes, we use BitConverter to get hex dash 551 # encoding (e.g. AE-09-3F) and they decode in python 552 code += " + [System.BitConverter]::ToString($out);" 553 stdout, stderr = _execute( 554 [powershell_exe, '-Command', code], 555 os.getcwd(), 556 re.compile(r'Unable to connect to|TLS|Internal Server Error'), 557 6 558 ) 559 if stdout[-2:] == b'\r\n' and b'\r\n\r\n' in stdout: 560 # An extra trailing crlf is added at the end by powershell 561 stdout = stdout[0:-2] 562 parts = stdout.split(b'\r\n\r\n', 1) 563 if len(parts) == 2: 564 stdout = parts[0] + b'\r\n\r\n' + codecs.decode(parts[1].replace(b'-', b''), 'hex_codec') 565 566 else: 567 args = [ 568 'curl', 569 '--request', 570 method, 571 '--location', 572 '--silent', 573 '--show-error', 574 '--include', 575 # Prevent curl from asking for an HTTP "100 Continue" response 576 '--header', 'Expect:' 577 ] 578 for key in headers: 579 args.append('--header') 580 args.append("%s: %s" % (key, headers[key])) 581 args.append('--data-binary') 582 args.append('@%s' % tempf_path) 583 args.append(url) 584 stdout, stderr = _execute( 585 args, 586 os.getcwd(), 587 re.compile(r'Failed to connect to|TLS|SSLRead|outstanding|cleanly'), 588 6 589 ) 590 finally: 591 if tempf_path and os.path.exists(tempf_path): 592 os.remove(tempf_path) 593 594 if len(stderr) > 0: 595 raise URLError("Error %sing %s:\n%s" % (method, url, stderr)) 596 597 parts = stdout.split(b'\r\n\r\n', 1) 598 if len(parts) != 2: 599 raise URLError("Error %sing %s, response data malformed:\n%s" % (method, url, stdout)) 600 header_block, body = parts 601 602 content_type_header = None 603 content_len_header = None 604 for hline in header_block.decode('iso-8859-1').splitlines(): 605 hline_parts = hline.split(':', 1) 606 if len(hline_parts) != 2: 607 continue 608 name, val = hline_parts 609 name = name.strip().lower() 610 val = val.strip() 611 if name == 'content-type': 612 content_type_header = val 613 if name == 'content-length': 614 content_len_header = val 615 616 if content_type_header is None and content_len_header != '0': 617 raise URLError("Error %sing %s, no content-type header:\n%s" % (method, url, stdout)) 618 619 if content_type_header is None: 620 content_type = 'text/plain' 621 encoding = 'utf-8' 622 else: 623 content_type, params = cgi.parse_header(content_type_header) 624 encoding = params.get('charset') 625 626 return (content_type, encoding, body) 627 628 629def _execute(params, cwd, retry=None, retries=0): 630 """ 631 Executes a subprocess 632 633 :param params: 634 A list of the executable and arguments to pass to it 635 636 :param cwd: 637 The working directory to execute the command in 638 639 :param retry: 640 If this string is present in stderr, or regex pattern matches stderr, retry the operation 641 642 :param retries: 643 An integer number of times to retry 644 645 :return: 646 A 2-element tuple of (stdout, stderr) 647 """ 648 649 proc = subprocess.Popen( 650 params, 651 stdout=subprocess.PIPE, 652 stderr=subprocess.PIPE, 653 cwd=cwd 654 ) 655 stdout, stderr = proc.communicate() 656 code = proc.wait() 657 if code != 0: 658 if retry and retries > 0: 659 stderr_str = stderr.decode('utf-8') 660 if isinstance(retry, Pattern): 661 if retry.search(stderr_str) is not None: 662 time.sleep(5) 663 return _execute(params, cwd, retry, retries - 1) 664 elif retry in stderr_str: 665 time.sleep(5) 666 return _execute(params, cwd, retry, retries - 1) 667 e = OSError('subprocess exit code for "%s" was %d: %s' % (' '.join(params), code, stderr)) 668 e.stdout = stdout 669 e.stderr = stderr 670 raise e 671 return (stdout, stderr) 672 673 674if __name__ == '__main__': 675 _codecov_submit() 676