1# Copyright 2016 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import argparse 6import json 7import os 8import sys 9 10# This is the list of tests to run. It is a dictionary with the following 11# fields: 12# 13# name (required): The name of the step, to show on the buildbot status page. 14# path (required): The path to the executable which runs the tests. 15# additional_args (optional): An array of optional arguments. 16# uses_app_engine_sdk (optional): True if app engine SDK must be in PYTHONPATH. 17# uses_sandbox_env (optional): True if CHROME_DEVEL_SANDBOX must be in 18# environment. 19# disabled (optional): List of platforms the test is disabled on. May contain 20# 'win', 'mac', or 'linux'. 21# outputs_presentation_json (optional): If True, pass in --presentation-json 22# argument to the test executable to allow it to update the buildbot status 23# page. More details here: 24# github.com/luci/recipes-py/blob/master/recipe_modules/generator_script/api.py 25_CATAPULT_TESTS = [ 26 { 27 'name': 'Build Python Tests', 28 'path': 'catapult_build/bin/run_py_tests', 29 }, 30 { 31 'name': 'Catapult Base Tests', 32 'path': 'catapult_base/bin/run_tests', 33 }, 34 { 35 'name': 'Dashboard Dev Server Tests Canary', 36 'path': 'dashboard/bin/run_dev_server_tests', 37 'additional_args': [ 38 '--no-install-hooks', 39 '--no-use-local-chrome', 40 '--channel=canary' 41 ], 42 # https://github.com/catapult-project/catapult/issues/2138 43 'disabled': ['linux', 'mac', 'win'], 44 'outputs_presentation_json': True, 45 }, 46 { 47 'name': 'Dashboard Dev Server Tests Stable', 48 'path': 'dashboard/bin/run_dev_server_tests', 49 'additional_args': [ 50 '--no-install-hooks', 51 '--no-use-local-chrome', 52 '--channel=stable', 53 ], 54 'outputs_presentation_json': True, 55 }, 56 { 57 'name': 'Dashboard Python Tests', 58 'path': 'dashboard/bin/run_py_tests', 59 'additional_args': ['--no-install-hooks'], 60 'uses_app_engine_sdk': True, 61 }, 62 { 63 'name': 'Dependency Manager Tests', 64 'path': 'dependency_manager/bin/run_tests', 65 }, 66 { 67 'name': 'Devil Python Tests', 68 'path': 'devil/bin/run_py_tests', 69 'disabled': ['mac', 'win'], 70 }, 71 { 72 'name': 'Perf Insights Dev Server Tests Canary', 73 'path': 'perf_insights/bin/run_dev_server_tests', 74 'additional_args': [ 75 '--no-install-hooks', 76 '--no-use-local-chrome', 77 '--channel=canary' 78 ], 79 # https://github.com/catapult-project/catapult/issues/2138 80 'disabled': ['linux', 'mac', 'win'], 81 'outputs_presentation_json': True, 82 }, 83 { 84 'name': 'Perf Insights Dev Server Tests Stable', 85 'path': 'perf_insights/bin/run_dev_server_tests', 86 'additional_args': [ 87 '--no-install-hooks', 88 '--no-use-local-chrome', 89 '--channel=stable', 90 ], 91 'uses_sandbox_env': True, 92 'outputs_presentation_json': True, 93 }, 94 { 95 'name': 'Perf Insights Python Tests', 96 'path': 'perf_insights/bin/run_py_tests', 97 'additional_args': ['--no-install-hooks'], 98 }, 99 { 100 'name': 'Perf VINN Insights Tests', 101 'path': 'perf_insights/bin/run_vinn_tests', 102 }, 103 { 104 'name': 'Py-vulcanize Tests', 105 'path': 'third_party/py_vulcanize/bin/run_py_tests', 106 'additional_args': ['--no-install-hooks'], 107 }, 108 { 109 'name': 'Systrace Tests', 110 'path': 'systrace/bin/run_tests', 111 }, 112 { 113 'name': 'Telemetry Tests with Stable Browser', 114 'path': 'telemetry/bin/run_tests', 115 'additional_args': [ 116 '--browser=reference', 117 '--start-xvfb' 118 ], 119 'uses_sandbox_env': True, 120 }, 121 { 122 'name': 'Tracing Dev Server Tests Canary', 123 'path': 'tracing/bin/run_dev_server_tests', 124 'additional_args': [ 125 '--no-install-hooks', 126 '--no-use-local-chrome', 127 '--channel=canary' 128 ], 129 # Test failing on Windows: 130 # https://github.com/catapult-project/catapult/issues/1816 131 # Tests failing on all platform: 132 # https://github.com/catapult-project/catapult/issues/2138 133 'disabled': ['win', 'linux', 'mac'], 134 'outputs_presentation_json': True, 135 }, 136 { 137 'name': 'Tracing Dev Server Tests Stable', 138 'path': 'tracing/bin/run_dev_server_tests', 139 'additional_args': [ 140 '--no-install-hooks', 141 '--no-use-local-chrome', 142 '--channel=stable', 143 ], 144 'outputs_presentation_json': True, 145 }, 146 { 147 'name': 'Tracing D8 Tests', 148 'path': 'tracing/bin/run_vinn_tests', 149 }, 150 { 151 'name': 'Tracing Python Tests', 152 'path': 'tracing/bin/run_py_tests', 153 'additional_args': ['--no-install-hooks'], 154 }, 155 { 156 'name': 'Vinn Tests', 157 'path': 'third_party/vinn/run_test', 158 }, 159] 160 161 162def main(args=None): 163 """Send list of test to run to recipes generator_script. 164 165 See documentation at: 166 github.com/luci/recipes-py/blob/master/recipe_modules/generator_script/api.py 167 """ 168 parser = argparse.ArgumentParser(description='Run catapult tests.') 169 parser.add_argument('--api-path-checkout', help='Path to catapult checkout') 170 parser.add_argument('--app-engine-sdk-pythonpath', 171 help='PYTHONPATH to include app engine SDK path') 172 parser.add_argument('--platform', 173 help='Platform name (linux, mac, or win)') 174 parser.add_argument('--output-json', help='Output for buildbot status page') 175 args = parser.parse_args(args) 176 177 steps = [] 178 for test in _CATAPULT_TESTS: 179 if args.platform in test.get('disabled', []): 180 continue 181 step = { 182 'name': test['name'], 183 'env': {} 184 } 185 step['cmd'] = ['python', os.path.join(args.api_path_checkout, test['path'])] 186 if test.get('additional_args'): 187 step['cmd'] += test['additional_args'] 188 if test.get('uses_app_engine_sdk'): 189 step['env']['PYTHONPATH'] = args.app_engine_sdk_pythonpath 190 if test.get('uses_sandbox_env'): 191 step['env']['CHROME_DEVEL_SANDBOX'] = '/opt/chromium/chrome_sandbox' 192 if test.get('outputs_presentation_json'): 193 step['outputs_presentation_json'] = True 194 steps.append(step) 195 with open(args.output_json, 'w') as outfile: 196 json.dump(steps, outfile) 197 198 199if __name__ == '__main__': 200 main(sys.argv[1:]) 201