1#!/usr/bin/env python 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Atest Argument Parser class for atest. 19""" 20 21import argparse 22 23import atest_utils 24import constants 25 26 27class AtestArgParser(argparse.ArgumentParser): 28 """Atest wrapper of ArgumentParser.""" 29 30 def __init__(self): 31 """Initialise an ArgumentParser instance.""" 32 atest_utils.print_data_collection_notice() 33 super(AtestArgParser, self).__init__( 34 description=constants.HELP_DESC, 35 epilog=self.EPILOG_TEXT, 36 formatter_class=argparse.RawTextHelpFormatter) 37 38 def add_atest_args(self): 39 """A function that does ArgumentParser.add_argument()""" 40 self.add_argument('tests', nargs='*', help='Tests to build and/or run.') 41 self.add_argument('-b', '--build', action='append_const', dest='steps', 42 const=constants.BUILD_STEP, help='Run a build.') 43 self.add_argument('-i', '--install', action='append_const', dest='steps', 44 const=constants.INSTALL_STEP, help='Install an APK.') 45 self.add_argument('--info', action='store_true', 46 help='Show module information.') 47 self.add_argument('--dry-run', action='store_true', 48 help='Dry run atest without building, installing and running ' 49 'tests in real.') 50 self.add_argument('-t', '--test', action='append_const', dest='steps', 51 const=constants.TEST_STEP, 52 help='Run the tests. WARNING: Many test configs force cleanup ' 53 'of device after test run. In this case, -d must be used in ' 54 'previous test run to disable cleanup, for -t to work. ' 55 'Otherwise, device will need to be setup again with -i.') 56 self.add_argument('-s', '--serial', help='The device to run the test on.') 57 self.add_argument('-L', '--list-modules', help='List testable modules for the given suite.') 58 self.add_argument('-d', '--disable-teardown', action='store_true', 59 help='Disables test teardown and cleanup.') 60 self.add_argument('-m', constants.REBUILD_MODULE_INFO_FLAG, action='store_true', 61 help='Forces a rebuild of the module-info.json file. ' 62 'This may be necessary following a repo sync or ' 63 'when writing a new test.') 64 self.add_argument('-w', '--wait-for-debugger', action='store_true', 65 help='Only for instrumentation tests. Waits for ' 66 'debugger prior to execution.') 67 self.add_argument('-v', '--verbose', action='store_true', 68 help='Display DEBUG level logging.') 69 self.add_argument('-a', '--all-abi', action='store_true', 70 help='Set to run tests for all abi.') 71 self.add_argument('--generate-baseline', nargs='?', type=int, const=5, default=0, 72 help='Generate baseline metrics, run 5 iterations by default. ' 73 'Provide an int argument to specify # iterations.') 74 self.add_argument('--generate-new-metrics', nargs='?', type=int, const=5, default=0, 75 help='Generate new metrics, run 5 iterations by default. ' 76 'Provide an int argument to specify # iterations.') 77 self.add_argument('--detect-regression', nargs='*', 78 help='Run regression detection algorithm. Supply ' 79 'path to baseline and/or new metrics folders.') 80 # Options related to module parameterization 81 self.add_argument('--instant', action='store_true', 82 help='Run the instant_app version of the module, ' 83 'if the module supports it. Note: running a test ' 84 'that does not support instant with --instant ' 85 'will result in nothing running.') 86 # Options related to Test Mapping 87 self.add_argument('-p', '--test-mapping', action='store_true', 88 help='Run tests in TEST_MAPPING files.') 89 self.add_argument('--include-subdirs', action='store_true', 90 help='Include tests in TEST_MAPPING files in sub directories.') 91 # Options related to deviceless testing. 92 self.add_argument('--host', action='store_true', 93 help='Run the test completely on the host without ' 94 'a device. (Note: running a host test that ' 95 'requires a device with --host will fail.)') 96 # This arg actually doesn't consume anything, it's primarily used for the 97 # help description and creating custom_args in the NameSpace object. 98 self.add_argument('--', dest='custom_args', nargs='*', 99 help='Specify custom args for the test runners. ' 100 'Everything after -- will be consumed as custom args.') 101 102 def get_args(self): 103 """This method is to get args from actions and return optional args. 104 105 Returns: 106 A list of optional arguments. 107 """ 108 argument_list = [] 109 # The output of _get_optional_actions(): [['-t', '--test'], [--info]] 110 # return an argument list: ['-t', '--test', '--info'] 111 for arg in self._get_optional_actions(): 112 argument_list.extend(arg.option_strings) 113 return argument_list 114 115 116 EPILOG_TEXT = ''' 117 118- - - - - - - - - 119IDENTIFYING TESTS 120- - - - - - - - - 121 122 The positional argument <tests> should be a reference to one or more 123 of the tests you'd like to run. Multiple tests can be run in one command by 124 separating test references with spaces. 125 126 Usage template: atest <reference_to_test_1> <reference_to_test_2> 127 128 A <reference_to_test> can be satisfied by the test's MODULE NAME, 129 MODULE:CLASS, CLASS NAME, TF INTEGRATION TEST, FILE PATH or PACKAGE NAME. 130 Explanations and examples of each follow. 131 132 133 < MODULE NAME > 134 135 Identifying a test by its module name will run the entire module. Input 136 the name as it appears in the LOCAL_MODULE or LOCAL_PACKAGE_NAME 137 variables in that test's Android.mk or Android.bp file. 138 139 Note: Use < TF INTEGRATION TEST > to run non-module tests integrated 140 directly into TradeFed. 141 142 Examples: 143 atest FrameworksServicesTests 144 atest CtsJankDeviceTestCases 145 146 147 < MODULE:CLASS > 148 149 Identifying a test by its class name will run just the tests in that 150 class and not the whole module. MODULE:CLASS is the preferred way to run 151 a single class. MODULE is the same as described above. CLASS is the 152 name of the test class in the .java file. It can either be the fully 153 qualified class name or just the basic name. 154 155 Examples: 156 atest FrameworksServicesTests:ScreenDecorWindowTests 157 atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests 158 atest CtsJankDeviceTestCases:CtsDeviceJankUi 159 160 161 < CLASS NAME > 162 163 A single class can also be run by referencing the class name without 164 the module name. 165 166 Examples: 167 atest ScreenDecorWindowTests 168 atest CtsDeviceJankUi 169 170 However, this will take more time than the equivalent MODULE:CLASS 171 reference, so we suggest using a MODULE:CLASS reference whenever 172 possible. Examples below are ordered by performance from the fastest 173 to the slowest: 174 175 Examples: 176 atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests 177 atest FrameworksServicesTests:ScreenDecorWindowTests 178 atest ScreenDecorWindowTests 179 180 < TF INTEGRATION TEST > 181 182 To run tests that are integrated directly into TradeFed (non-modules), 183 input the name as it appears in the output of the "tradefed.sh list 184 configs" cmd. 185 186 Examples: 187 atest example/reboot 188 atest native-benchmark 189 190 191 < FILE PATH > 192 193 Both module-based tests and integration-based tests can be run by 194 inputting the path to their test file or dir as appropriate. A single 195 class can also be run by inputting the path to the class's java file. 196 Both relative and absolute paths are supported. 197 198 Example - 2 ways to run the `CtsJankDeviceTestCases` module via path: 199 1. run module from android <repo root>: 200 atest cts/tests/jank/jank 201 202 2. from <android root>/cts/tests/jank: 203 atest . 204 205 Example - run a specific class within CtsJankDeviceTestCases module 206 from <android repo> root via path: 207 atest cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java 208 209 Example - run an integration test from <android repo> root via path: 210 atest tools/tradefederation/contrib/res/config/example/reboot.xml 211 212 213 < PACKAGE NAME > 214 215 Atest supports searching tests from package name as well. 216 217 Examples: 218 atest com.android.server.wm 219 atest android.jank.cts 220 221 222- - - - - - - - - - - - - - - - - - - - - - - - - - 223SPECIFYING INDIVIDUAL STEPS: BUILD, INSTALL OR RUN 224- - - - - - - - - - - - - - - - - - - - - - - - - - 225 226 The -b, -i and -t options allow you to specify which steps you want to run. 227 If none of those options are given, then all steps are run. If any of these 228 options are provided then only the listed steps are run. 229 230 Note: -i alone is not currently support and can only be included with -t. 231 Both -b and -t can be run alone. 232 233 Examples: 234 atest -b <test> (just build targets) 235 atest -t <test> (run tests only) 236 atest -it <test> (install apk and run tests) 237 atest -bt <test> (build targets, run tests, but skip installing apk) 238 239 240 Atest now has the ability to force a test to skip its cleanup/teardown step. 241 Many tests, e.g. CTS, cleanup the device after the test is run, so trying to 242 rerun your test with -t will fail without having the --disable-teardown 243 parameter. Use -d before -t to skip the test clean up step and test iteratively. 244 245 atest -d <test> (disable installing apk and cleanning up device) 246 atest -t <test> 247 248 Note that -t disables both setup/install and teardown/cleanup of the 249 device. So you can continue to rerun your test with just 250 251 atest -t <test> 252 253 as many times as you want. 254 255 256- - - - - - - - - - - - - 257RUNNING SPECIFIC METHODS 258- - - - - - - - - - - - - 259 260 It is possible to run only specific methods within a test class. To run 261 only specific methods, identify the class in any of the ways supported for 262 identifying a class (MODULE:CLASS, FILE PATH, etc) and then append the 263 name of the method or method using the following template: 264 265 <reference_to_class>#<method1> 266 267 Multiple methods can be specified with commas: 268 269 <reference_to_class>#<method1>,<method2>,<method3>... 270 271 Examples: 272 atest com.android.server.wm.ScreenDecorWindowTests#testMultipleDecors 273 274 atest FrameworksServicesTests:ScreenDecorWindowTests#testFlagChange,testRemoval 275 276 277- - - - - - - - - - - - - 278RUNNING MULTIPLE CLASSES 279- - - - - - - - - - - - - 280 281 To run multiple classes, deliminate them with spaces just like you would 282 when running multiple tests. Atest will handle building and running 283 classes in the most efficient way possible, so specifying a subset of 284 classes in a module will improve performance over running the whole module. 285 286 287 Examples: 288 - two classes in same module: 289 atest FrameworksServicesTests:ScreenDecorWindowTests FrameworksServicesTests:DimmerTests 290 291 - two classes, different modules: 292 atest FrameworksServicesTests:ScreenDecorWindowTests CtsJankDeviceTestCases:CtsDeviceJankUi 293 294 295- - - - - - - - - - - 296REGRESSION DETECTION 297- - - - - - - - - - - 298 299 Generate pre-patch or post-patch metrics without running regression detection: 300 301 Example: 302 atest <test> --generate-baseline <optional iter> 303 atest <test> --generate-new-metrics <optional iter> 304 305 Local regression detection can be run in three options: 306 307 1) Provide a folder containing baseline (pre-patch) metrics (generated 308 previously). Atest will run the tests n (default 5) iterations, generate 309 a new set of post-patch metrics, and compare those against existing metrics. 310 311 Example: 312 atest <test> --detect-regression </path/to/baseline> --generate-new-metrics <optional iter> 313 314 2) Provide a folder containing post-patch metrics (generated previously). 315 Atest will run the tests n (default 5) iterations, generate a new set of 316 pre-patch metrics, and compare those against those provided. Note: the 317 developer needs to revert the device/tests to pre-patch state to generate 318 baseline metrics. 319 320 Example: 321 atest <test> --detect-regression </path/to/new> --generate-baseline <optional iter> 322 323 3) Provide 2 folders containing both pre-patch and post-patch metrics. Atest 324 will run no tests but the regression detection algorithm. 325 326 Example: 327 atest --detect-regression </path/to/baseline> </path/to/new> 328 329 330- - - - - - - - - - - - 331TESTS IN TEST MAPPING 332- - - - - - - - - - - - 333 334 Atest can run tests in TEST_MAPPING files: 335 336 1) Run presubmit tests in TEST_MAPPING files in current and parent 337 directories. You can also specify a target directory. 338 339 Example: 340 atest (run presubmit tests in TEST_MAPPING files in current and parent directories) 341 atest --test-mapping </path/to/project> 342 (run presubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories) 343 344 2) Run a specified test group in TEST_MAPPING files. 345 346 Example: 347 atest :postsubmit 348 (run postsubmit tests in TEST_MAPPING files in current and parent directories) 349 atest :all 350 (Run tests from all groups in TEST_MAPPING files) 351 atest --test-mapping </path/to/project>:postsubmit 352 (run postsubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories) 353 354 3) Run tests in TEST_MAPPING files including sub directories 355 356 By default, atest will only search for tests in TEST_MAPPING files in 357 current (or given directory) and its parent directories. If you want to run 358 tests in TEST_MAPPING files in the sub-directories, you can use option 359 --include-subdirs to force atest to include those tests too. 360 361 Example: 362 atest --include-subdirs [optional </path/to/project>:<test_group_name>] 363 (run presubmit tests in TEST_MAPPING files in current, sub and parent directories) 364 A path can be provided optionally if you want to search for tests in a give 365 directory, with optional test group name. By default, the test group is 366 presubmit. 367 368''' 369