1# Copyright 2016 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import argparse 6import json 7import os 8import sys 9 10# This is the list of tests to run. It is a dictionary with the following 11# fields: 12# 13# name (required): The name of the step, to show on the buildbot status page. 14# path (required): The path to the executable which runs the tests. 15# additional_args (optional): An array of optional arguments. 16# uses_app_engine_sdk (optional): True if app engine SDK must be in PYTHONPATH. 17# uses_sandbox_env (optional): True if CHROME_DEVEL_SANDBOX must be in 18# environment. 19# disabled (optional): List of platforms the test is disabled on. May contain 20# 'win', 'mac', 'linux', or 'android'. 21# outputs_presentation_json (optional): If True, pass in --presentation-json 22# argument to the test executable to allow it to update the buildbot status 23# page. More details here: 24# github.com/luci/recipes-py/blob/master/recipe_modules/generator_script/api.py 25_CATAPULT_TESTS = [ 26 { 27 'name': 'BattOr Smoke Tests', 28 'path': 'common/battor/battor/battor_wrapper_devicetest.py', 29 'disabled': ['android'], 30 }, 31 { 32 'name': 'BattOr Unit Tests', 33 'path': 'common/battor/bin/run_py_tests', 34 'disabled': ['android'], 35 }, 36 { 37 'name': 'Build Python Tests', 38 'path': 'catapult_build/bin/run_py_tests', 39 'disabled': ['android'], 40 }, 41 { 42 'name': 'Common Tests', 43 'path': 'common/bin/run_tests', 44 }, 45 { 46 'name': 'Dashboard Dev Server Tests Canary', 47 'path': 'dashboard/bin/run_dev_server_tests', 48 'additional_args': [ 49 '--no-install-hooks', 50 '--no-use-local-chrome', 51 '--channel=canary' 52 ], 53 'outputs_presentation_json': True, 54 'disabled': ['android'], 55 }, 56 { 57 'name': 'Dashboard Dev Server Tests Stable', 58 'path': 'dashboard/bin/run_dev_server_tests', 59 'additional_args': [ 60 '--no-install-hooks', 61 '--no-use-local-chrome', 62 '--channel=stable', 63 ], 64 'outputs_presentation_json': True, 65 'disabled': ['android'], 66 }, 67 { 68 'name': 'Dashboard Python Tests', 69 'path': 'dashboard/bin/run_py_tests', 70 'additional_args': ['--no-install-hooks'], 71 'uses_app_engine_sdk': True, 72 'disabled': ['android'], 73 }, 74 { 75 'name': 'Dependency Manager Tests', 76 'path': 'dependency_manager/bin/run_tests', 77 }, 78 { 79 'name': 'Devil Device Tests', 80 'path': 'devil/bin/run_py_devicetests', 81 'disabled': ['win', 'mac', 'linux'] 82 }, 83 { 84 'name': 'Devil Python Tests', 85 'path': 'devil/bin/run_py_tests', 86 'disabled': ['mac', 'win'], 87 }, 88 { 89 'name': 'Node Smoke Test', 90 'path': 'common/node_runner/bin/test_node_for_smoke', 91 'disabled': ['android'], 92 }, 93 { 94 'name': 'Py-vulcanize Tests', 95 'path': 'third_party/py_vulcanize/bin/run_py_tests', 96 'additional_args': ['--no-install-hooks'], 97 'disabled': ['android'], 98 }, 99 { 100 'name': 'Systrace Tests', 101 'path': 'systrace/bin/run_tests', 102 }, 103 { 104 'name': 'Telemetry Tests with Stable Browser', 105 'path': 'telemetry/bin/run_tests', 106 'additional_args': [ 107 '--browser=reference', 108 '--start-xvfb' 109 ], 110 'uses_sandbox_env': True, 111 'disabled': ['android'], 112 }, 113 { 114 'name': 'Telemetry Integration Tests with Stable Browser', 115 'path': 'telemetry/bin/run_browser_tests', 116 'additional_args': [ 117 'SimpleBrowserTest', 118 '--browser=reference', 119 ], 120 'uses_sandbox_env': True, 121 'disabled': ['android', 'linux'], # TODO(nedn): enable this on linux 122 }, 123 { 124 'name': 'Tracing Dev Server Tests Canary', 125 'path': 'tracing/bin/run_dev_server_tests', 126 'additional_args': [ 127 '--no-install-hooks', 128 '--no-use-local-chrome', 129 '--channel=canary' 130 ], 131 'outputs_presentation_json': True, 132 'disabled': ['android'], 133 }, 134 { 135 'name': 'Tracing Dev Server Tests Stable', 136 'path': 'tracing/bin/run_dev_server_tests', 137 'additional_args': [ 138 '--no-install-hooks', 139 '--no-use-local-chrome', 140 '--channel=stable', 141 ], 142 'outputs_presentation_json': True, 143 'disabled': ['android'], 144 }, 145 { 146 'name': 'Tracing D8 Tests', 147 'path': 'tracing/bin/run_vinn_tests', 148 'disabled': ['android'], 149 }, 150 { 151 'name': 'Tracing Python Tests', 152 'path': 'tracing/bin/run_py_tests', 153 'additional_args': ['--no-install-hooks'], 154 'disabled': ['android'], 155 }, 156 { 157 'name': 'Vinn Tests', 158 'path': 'third_party/vinn/bin/run_tests', 159 'disabled': ['android'], 160 }, 161 { 162 'name': 'NetLog Viewer Dev Server Tests', 163 'path': 'netlog_viewer/bin/run_dev_server_tests', 164 'additional_args': [ 165 '--no-install-hooks', 166 '--no-use-local-chrome', 167 ], 168 'disabled': ['android', 'win', 'mac', 'linux'], 169 }, 170] 171 172 173def main(args=None): 174 """Send list of test to run to recipes generator_script. 175 176 See documentation at: 177 github.com/luci/recipes-py/blob/master/recipe_modules/generator_script/api.py 178 """ 179 parser = argparse.ArgumentParser(description='Run catapult tests.') 180 parser.add_argument('--api-path-checkout', help='Path to catapult checkout') 181 parser.add_argument('--app-engine-sdk-pythonpath', 182 help='PYTHONPATH to include app engine SDK path') 183 parser.add_argument('--platform', 184 help='Platform name (linux, mac, or win)') 185 parser.add_argument('--output-json', help='Output for buildbot status page') 186 args = parser.parse_args(args) 187 188 steps = [{ 189 # Always remove stale pyc files first. Not listed as a test above 190 # because it is a step and not a test, and must be first. 191 'name': 'Remove Stale PYC files', 192 'cmd': ['python', 193 os.path.join(args.api_path_checkout, 194 'catapult_build', 'remove_stale_pyc_files.py'), 195 args.api_path_checkout] 196 }] 197 if args.platform == 'android': 198 # On Android, we need to prepare the devices a bit before using them in 199 # tests. These steps are not listed as tests above because they aren't 200 # tests and because they must precede all tests. 201 steps.extend([ 202 { 203 'name': 'Android: Recover Devices', 204 'cmd': ['python', 205 os.path.join(args.api_path_checkout, 'devil', 'devil', 206 'android', 'tools', 'device_recovery.py')], 207 }, 208 { 209 'name': 'Android: Provision Devices', 210 'cmd': ['python', 211 os.path.join(args.api_path_checkout, 'devil', 'devil', 212 'android', 'tools', 'provision_devices.py')], 213 }, 214 { 215 'name': 'Android: Device Status', 216 'cmd': ['python', 217 os.path.join(args.api_path_checkout, 'devil', 'devil', 218 'android', 'tools', 'device_status.py')], 219 }, 220 ]) 221 222 for test in _CATAPULT_TESTS: 223 if args.platform in test.get('disabled', []): 224 continue 225 step = { 226 'name': test['name'], 227 'env': {} 228 } 229 step['cmd'] = ['python', os.path.join(args.api_path_checkout, test['path'])] 230 if step['name'] == 'Systrace Tests': 231 step['cmd'] += ['--device=' + args.platform] 232 if test.get('additional_args'): 233 step['cmd'] += test['additional_args'] 234 if test.get('uses_app_engine_sdk'): 235 step['env']['PYTHONPATH'] = args.app_engine_sdk_pythonpath 236 if test.get('uses_sandbox_env'): 237 step['env']['CHROME_DEVEL_SANDBOX'] = '/opt/chromium/chrome_sandbox' 238 if test.get('outputs_presentation_json'): 239 step['outputs_presentation_json'] = True 240 steps.append(step) 241 with open(args.output_json, 'w') as outfile: 242 json.dump(steps, outfile) 243 244 245if __name__ == '__main__': 246 main(sys.argv[1:]) 247