1# -*- coding: utf-8 -*- 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Setting files for global, benchmark and labels.""" 7 8from __future__ import print_function 9 10from field import BooleanField 11from field import EnumField 12from field import FloatField 13from field import IntegerField 14from field import ListField 15from field import TextField 16from settings import Settings 17 18 19class BenchmarkSettings(Settings): 20 """Settings used to configure individual benchmarks.""" 21 22 def __init__(self, name): 23 super(BenchmarkSettings, self).__init__(name, 'benchmark') 24 self.AddField( 25 TextField('test_name', 26 description='The name of the test to run. ' 27 'Defaults to the name of the benchmark.')) 28 self.AddField( 29 TextField('test_args', 30 description='Arguments to be passed to the ' 31 'test.')) 32 self.AddField( 33 IntegerField( 34 'iterations', 35 required=False, 36 default=0, 37 description='Number of iterations to run the test. ' 38 'If not set, will run each benchmark test the optimum number of ' 39 'times to get a stable result.')) 40 self.AddField( 41 TextField('suite', 42 default='test_that', 43 description='The type of the benchmark.')) 44 self.AddField( 45 IntegerField('retries', 46 default=0, 47 description='Number of times to retry a ' 48 'benchmark run.')) 49 self.AddField( 50 BooleanField('run_local', 51 description='Run benchmark harness on the DUT. ' 52 'Currently only compatible with the suite: ' 53 'telemetry_Crosperf.', 54 required=False, 55 default=True)) 56 self.AddField( 57 FloatField( 58 'weight', 59 default=0.0, 60 description='Weight of the benchmark for CWP approximation')) 61 62 63class LabelSettings(Settings): 64 """Settings for each label.""" 65 66 def __init__(self, name): 67 super(LabelSettings, self).__init__(name, 'label') 68 self.AddField( 69 TextField('chromeos_image', 70 required=False, 71 description='The path to the image to run tests ' 72 'on, for local/custom-built images. See the ' 73 "'build' option for official or trybot images.")) 74 self.AddField( 75 TextField( 76 'autotest_path', 77 required=False, 78 description='Autotest directory path relative to chroot which ' 79 'has autotest files for the image to run tests requiring autotest ' 80 'files.')) 81 self.AddField( 82 TextField( 83 'debug_path', 84 required=False, 85 description='Debug info directory relative to chroot which has ' 86 'symbols and vmlinux that can be used by perf tool.')) 87 self.AddField( 88 TextField('chromeos_root', 89 description='The path to a chromeos checkout which ' 90 'contains a src/scripts directory. Defaults to ' 91 'the chromeos checkout which contains the ' 92 'chromeos_image.')) 93 self.AddField( 94 ListField('remote', 95 description='A comma-separated list of IPs of chromeos' 96 'devices to run experiments on.')) 97 self.AddField( 98 TextField('image_args', 99 required=False, 100 default='', 101 description='Extra arguments to pass to ' 102 'image_chromeos.py.')) 103 self.AddField( 104 TextField('cache_dir', 105 default='', 106 description='The cache dir for this image.')) 107 self.AddField( 108 TextField('compiler', 109 default='gcc', 110 description='The compiler used to build the ' 111 'ChromeOS image (gcc or llvm).')) 112 self.AddField( 113 TextField('chrome_src', 114 description='The path to the source of chrome. ' 115 'This is used to run telemetry benchmarks. ' 116 'The default one is the src inside chroot.', 117 required=False, 118 default='')) 119 self.AddField( 120 TextField('build', 121 description='The xbuddy specification for an ' 122 'official or trybot image to use for tests. ' 123 "'/remote' is assumed, and the board is given " 124 "elsewhere, so omit the '/remote/<board>/' xbuddy " 125 'prefix.', 126 required=False, 127 default='')) 128 129 130class GlobalSettings(Settings): 131 """Settings that apply per-experiment.""" 132 133 def __init__(self, name): 134 super(GlobalSettings, self).__init__(name, 'global') 135 self.AddField( 136 TextField('name', 137 description='The name of the experiment. Just an ' 138 'identifier.')) 139 self.AddField( 140 TextField('board', 141 description='The target board for running ' 142 'experiments on, e.g. x86-alex.')) 143 self.AddField( 144 BooleanField('crosfleet', 145 description='Whether to run experiments via crosfleet.', 146 default=False)) 147 self.AddField( 148 ListField('remote', 149 description='A comma-separated list of IPs of ' 150 'chromeos devices to run experiments on.')) 151 self.AddField( 152 BooleanField('rerun_if_failed', 153 description='Whether to re-run failed test runs ' 154 'or not.', 155 default=False)) 156 self.AddField( 157 BooleanField('rm_chroot_tmp', 158 default=False, 159 description='Whether to remove the test_that ' 160 'result in the chroot.')) 161 self.AddField( 162 ListField('email', 163 description='Space-separated list of email ' 164 'addresses to send email to.')) 165 self.AddField( 166 BooleanField('rerun', 167 description='Whether to ignore the cache and ' 168 'for tests to be re-run.', 169 default=False)) 170 self.AddField( 171 BooleanField('same_specs', 172 default=True, 173 description='Ensure cached runs are run on the ' 174 'same kind of devices which are specified as a ' 175 'remote.')) 176 self.AddField( 177 BooleanField('same_machine', 178 default=False, 179 description='Ensure cached runs are run on the ' 180 'same remote.')) 181 self.AddField( 182 BooleanField('use_file_locks', 183 default=False, 184 description='DEPRECATED: Whether to use the file locks ' 185 'or AFE server lock mechanism.')) 186 self.AddField( 187 IntegerField( 188 'iterations', 189 required=False, 190 default=0, 191 description='Number of iterations to run all tests. ' 192 'If not set, will run each benchmark test the optimum number of ' 193 'times to get a stable result.')) 194 self.AddField( 195 TextField('chromeos_root', 196 description='The path to a chromeos checkout which ' 197 'contains a src/scripts directory. Defaults to ' 198 'the chromeos checkout which contains the ' 199 'chromeos_image.')) 200 self.AddField( 201 TextField('logging_level', 202 default='average', 203 description='The level of logging desired. ' 204 "Options are 'quiet', 'average', and 'verbose'.")) 205 self.AddField( 206 IntegerField('acquire_timeout', 207 default=0, 208 description='Number of seconds to wait for ' 209 'machine before exit if all the machines in ' 210 'the experiment file are busy. Default is 0.')) 211 self.AddField( 212 TextField('perf_args', 213 default='', 214 description='The optional profile command. It ' 215 'enables perf commands to record perforamance ' 216 'related counters. It must start with perf ' 217 'command record or stat followed by arguments.')) 218 self.AddField( 219 BooleanField('download_debug', 220 default=True, 221 description='Download compressed debug symbols alongwith ' 222 'image. This can provide more info matching symbols for' 223 'profiles, but takes larger space. By default, download' 224 'it only when perf_args is specified.')) 225 self.AddField( 226 TextField('cache_dir', 227 default='', 228 description='The abs path of cache dir. ' 229 'Default is /home/$(whoami)/cros_scratch.')) 230 self.AddField( 231 BooleanField('cache_only', 232 default=False, 233 description='Whether to use only cached ' 234 'results (do not rerun failed tests).')) 235 self.AddField( 236 BooleanField('no_email', 237 default=False, 238 description='Whether to disable the email to ' 239 'user after crosperf finishes.')) 240 self.AddField( 241 BooleanField('json_report', 242 default=False, 243 description='Whether to generate a json version ' 244 'of the report, for archiving.')) 245 self.AddField( 246 BooleanField('show_all_results', 247 default=False, 248 description='When running Telemetry tests, ' 249 'whether to all the results, instead of just ' 250 'the default (summary) results.')) 251 self.AddField( 252 TextField('share_cache', 253 default='', 254 description='Path to alternate cache whose data ' 255 'you want to use. It accepts multiple directories ' 256 'separated by a ",".')) 257 self.AddField( 258 TextField('results_dir', default='', description='The results dir.')) 259 self.AddField( 260 BooleanField( 261 'compress_results', 262 default=True, 263 description='Whether to compress all test results other than ' 264 'reports into a tarball to save disk space.')) 265 self.AddField( 266 TextField('locks_dir', 267 default='', 268 description='An alternate directory to use for ' 269 'storing/checking machine file locks for local machines. ' 270 'By default the file locks directory is ' 271 '/google/data/rw/users/mo/mobiletc-prebuild/locks.\n' 272 'WARNING: If you use your own locks directory, ' 273 'there is no guarantee that someone else might not ' 274 'hold a lock on the same machine in a different ' 275 'locks directory.')) 276 self.AddField( 277 TextField('chrome_src', 278 description='The path to the source of chrome. ' 279 'This is used to run telemetry benchmarks. ' 280 'The default one is the src inside chroot.', 281 required=False, 282 default='')) 283 self.AddField( 284 IntegerField('retries', 285 default=0, 286 description='Number of times to retry a ' 287 'benchmark run.')) 288 self.AddField( 289 TextField('cwp_dso', 290 description='The DSO type that we want to use for ' 291 'CWP approximation. This is used to run telemetry ' 292 'benchmarks. Valid DSO types can be found from dso_list ' 293 'in experiment_factory.py. The default value is set to ' 294 'be empty.', 295 required=False, 296 default='')) 297 self.AddField( 298 BooleanField('enable_aslr', 299 description='Enable ASLR on the machine to run the ' 300 'benchmarks. ASLR is disabled by default', 301 required=False, 302 default=False)) 303 self.AddField( 304 BooleanField('ignore_min_max', 305 description='When doing math for the raw results, ' 306 'ignore min and max values to reduce noise.', 307 required=False, 308 default=False)) 309 self.AddField( 310 TextField( 311 'intel_pstate', 312 description='Intel Pstate mode.\n' 313 'Supported modes: "active", "passive", "no_hwp".\n' 314 'Default is "no_hwp" which disables hardware pstates to avoid ' 315 'noise in benchmarks.', 316 required=False, 317 default='no_hwp')) 318 self.AddField( 319 BooleanField('turbostat', 320 description='Run turbostat process in the background' 321 ' of a benchmark. Enabled by default.', 322 required=False, 323 default=True)) 324 self.AddField( 325 FloatField( 326 'top_interval', 327 description='Run top command in the background of a benchmark with' 328 ' interval of sampling specified in seconds.\n' 329 'Recommended values 1-5. Lower number provides more accurate' 330 ' data.\n' 331 'With 0 - do not run top.\n' 332 'NOTE: Running top with interval 1-5 sec has insignificant' 333 ' performance impact (performance degradation does not exceed' 334 ' 0.3%%, measured on x86_64, ARM32, and ARM64). ' 335 'The default value is 1.', 336 required=False, 337 default=1)) 338 self.AddField( 339 IntegerField('cooldown_temp', 340 required=False, 341 default=40, 342 description='Wait until CPU temperature goes down below' 343 ' specified temperature in Celsius' 344 ' prior starting a benchmark. ' 345 'By default the value is set to 40 degrees.')) 346 self.AddField( 347 IntegerField('cooldown_time', 348 required=False, 349 default=10, 350 description='Wait specified time in minutes allowing' 351 ' CPU to cool down. Zero value disables cooldown. ' 352 'The default value is 10 minutes.')) 353 self.AddField( 354 EnumField( 355 'governor', 356 options=[ 357 'performance', 358 'powersave', 359 'userspace', 360 'ondemand', 361 'conservative', 362 'schedutils', 363 'sched', 364 'interactive', 365 ], 366 default='performance', 367 required=False, 368 description='Setup CPU governor for all cores.\n' 369 'For more details refer to:\n' 370 'https://www.kernel.org/doc/Documentation/cpu-freq/governors.txt. ' 371 'Default is "performance" governor.')) 372 self.AddField( 373 EnumField( 374 'cpu_usage', 375 options=[ 376 'all', 377 'big_only', 378 'little_only', 379 'exclusive_cores', 380 ], 381 default='all', 382 required=False, 383 description='Restrict usage of CPUs to decrease CPU interference.\n' 384 '"all" - no restrictions;\n' 385 '"big-only", "little-only" - enable only big/little cores,' 386 ' applicable only on ARM;\n' 387 '"exclusive-cores" - (for future use)' 388 ' isolate cores for exclusive use of benchmark processes. ' 389 'By default use all CPUs.')) 390 self.AddField( 391 IntegerField( 392 'cpu_freq_pct', 393 required=False, 394 default=95, 395 description='Setup CPU frequency to a supported value less than' 396 ' or equal to a percent of max_freq. ' 397 'CPU frequency is reduced to 95%% by default to reduce thermal ' 398 'throttling.')) 399 self.AddField( 400 BooleanField( 401 'no_lock', 402 default=False, 403 description='Do not attempt to lock the DUT.' 404 ' Useful when lock is held externally, say with crosfleet.')) 405 406 407class SettingsFactory(object): 408 """Factory class for building different types of Settings objects. 409 410 This factory is currently hardcoded to produce settings for ChromeOS 411 experiment files. The idea is that in the future, other types 412 of settings could be produced. 413 """ 414 415 def GetSettings(self, name, settings_type): 416 if settings_type == 'label' or not settings_type: 417 return LabelSettings(name) 418 if settings_type == 'global': 419 return GlobalSettings(name) 420 if settings_type == 'benchmark': 421 return BenchmarkSettings(name) 422 423 raise TypeError("Invalid settings type: '%s'." % settings_type) 424