1# SPDX-License-Identifier: Apache-2.0 2# 3# Copyright (C) 2015, ARM Limited and contributors. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); you may 6# not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import argparse 19import logging 20import os 21import select 22 23from subprocess import Popen, PIPE 24from time import sleep 25 26from conf import LisaLogging 27from android import System, Workload 28from env import TestEnv 29 30from devlib.utils.misc import memoized 31from devlib.utils.android import fastboot_command 32 33class LisaBenchmark(object): 34 """ 35 A base class for LISA custom benchmarks execution 36 37 This class is intended to be subclassed in order to create a custom 38 benckmark execution for LISA. 39 It sets up the TestEnv and and provides convenience methods for 40 test environment setup, execution and post-processing. 41 42 Subclasses should provide a bm_conf to setup the TestEnv and 43 a set of optional callback methods to configuere a test environment 44 and process collected data. 45 46 Example users of this class can be found under LISA's tests/benchmarks 47 directory. 48 """ 49 50 bm_conf = { 51 52 # Target platform and board 53 "platform" : 'android', 54 55 # Define devlib modules to load 56 "modules" : [ 57 'cpufreq', 58 'cpuidle', 59 ], 60 61 # FTrace events to collect for all the tests configuration which have 62 # the "ftrace" flag enabled 63 "ftrace" : { 64 "events" : [ 65 "sched_switch", 66 "sched_overutilized", 67 "sched_contrib_scale_f", 68 "sched_load_avg_cpu", 69 "sched_load_avg_task", 70 "sched_tune_tasks_update", 71 "sched_boost_cpu", 72 "sched_boost_task", 73 "sched_energy_diff", 74 "cpu_frequency", 75 "cpu_idle", 76 "cpu_capacity", 77 ], 78 "buffsize" : 10 * 1024, 79 }, 80 81 # Default EnergyMeter Configuration 82 "emeter" : { 83 "instrument" : "monsoon", 84 "conf" : { } 85 }, 86 87 # Tools required by the experiments 88 "tools" : [ 'trace-cmd' ], 89 90 } 91 """Override this with a dictionary or JSON path to configure the TestEnv""" 92 93 bm_name = None 94 """Override this with the name of the LISA's benchmark to run""" 95 96 bm_params = None 97 """Override this with the set of parameters for the LISA's benchmark to run""" 98 99 bm_collect = None 100 """Override this with the set of data to collect during test exeution""" 101 102 bm_reboot = False 103 """Override this with True if a boot image was passed as command line parameter""" 104 105 bm_iterations = 1 106 """Override this with the desired number of iterations of the test""" 107 108 bm_iterations_pause = 30 109 """ 110 Override this with the desired amount of time (in seconds) to pause 111 for before each iteration 112 """ 113 114 bm_iterations_reboot = False 115 """ 116 Override this with the desired behaviour: reboot or not reboot before 117 each iteration 118 """ 119 120 def benchmarkInit(self): 121 """ 122 Code executed before running the benchmark 123 """ 124 pass 125 126 def benchmarkFinalize(self): 127 """ 128 Code executed after running the benchmark 129 """ 130 pass 131 132################################################################################ 133# Private Interface 134 135 @memoized 136 def _parseCommandLine(self): 137 138 parser = argparse.ArgumentParser( 139 description='LISA Benchmark Configuration') 140 141 # Bootup settings 142 parser.add_argument('--boot-image', type=str, 143 default=None, 144 help='Path of the Android boot.img to be used') 145 parser.add_argument('--boot-timeout', type=int, 146 default=60, 147 help='Timeout in [s] to wait after a reboot (default 60)') 148 149 # Android settings 150 parser.add_argument('--android-device', type=str, 151 default=None, 152 help='Identifier of the Android target to use') 153 parser.add_argument('--android-home', type=str, 154 default=None, 155 help='Path used to configure ANDROID_HOME') 156 157 # Test customization 158 parser.add_argument('--results-dir', type=str, 159 default=self.__class__.__name__, 160 help='Results folder, ' 161 'if specified override test defaults') 162 parser.add_argument('--collect', type=str, 163 default=None, 164 help='Set of metrics to collect, ' 165 'e.g. "energy systrace_30" to sample energy and collect a 30s systrace, ' 166 'if specified overrides test defaults') 167 parser.add_argument('--iterations', type=int, 168 default=1, 169 help='Number of iterations the same test has to be repeated for (default 1)') 170 parser.add_argument('--iterations-pause', type=int, 171 default=30, 172 help='Amount of time (in seconds) to pause for before each iteration (default 30s)') 173 parser.add_argument('--iterations-reboot', action="store_true", 174 help='Reboot before each iteration (default False)') 175 176 # Measurements settings 177 parser.add_argument('--iio-channel-map', type=str, 178 default=None, 179 help='List of IIO channels to sample, ' 180 'e.g. "ch0:0,ch3:1" to sample CHs 0 and 3, ' 181 'if specified overrides test defaults') 182 183 # Parse command line arguments 184 return parser.parse_args() 185 186 187 def _getBmConf(self): 188 # Override default configuration with command line parameters 189 if self.args.boot_image: 190 self.bm_reboot = True 191 if self.args.android_device: 192 self.bm_conf['device'] = self.args.android_device 193 if self.args.android_home: 194 self.bm_conf['ANDROID_HOME'] = self.args.android_home 195 if self.args.results_dir: 196 self.bm_conf['results_dir'] = self.args.results_dir 197 if self.args.collect: 198 self.bm_collect = self.args.collect 199 if self.args.iterations: 200 self.bm_iterations = self.args.iterations 201 if self.args.iterations_pause: 202 self.bm_iterations_pause = self.args.iterations_pause 203 if self.args.iterations_reboot: 204 self.bm_iterations_reboot = True 205 206 # Override energy meter configuration 207 if self.args.iio_channel_map: 208 em = { 209 'instrument' : 'acme', 210 'channel_map' : {}, 211 } 212 for ch in self.args.iio_channel_map.split(','): 213 ch_name, ch_id = ch.split(':') 214 em['channel_map'][ch_name] = ch_id 215 self.bm_conf['emeter'] = em 216 self._log.info('Using ACME energy meter channels: %s', em) 217 218 # Override EM if energy collection not required 219 if 'energy' not in self.bm_collect: 220 try: 221 self.bm_conf.pop('emeter') 222 except: 223 pass 224 225 return self.bm_conf 226 227 def _getWorkload(self): 228 if self.bm_name is None: 229 msg = 'Benchmark subclasses must override the `bm_name` attribute' 230 raise NotImplementedError(msg) 231 # Get a referench to the worload to run 232 wl = Workload.getInstance(self.te, self.bm_name) 233 if wl is None: 234 raise ValueError('Specified benchmark [{}] is not supported'\ 235 .format(self.bm_name)) 236 return wl 237 238 def _getBmParams(self): 239 if self.bm_params is None: 240 msg = 'Benchmark subclasses must override the `bm_params` attribute' 241 raise NotImplementedError(msg) 242 return self.bm_params 243 244 def _getBmCollect(self): 245 if self.bm_collect is None: 246 msg = 'Benchmark subclasses must override the `bm_collect` attribute' 247 self._log.warning(msg) 248 return '' 249 return self.bm_collect 250 251 def _preInit(self): 252 """ 253 Code executed before running the benchmark 254 """ 255 # If iterations_reboot is True we are going to reboot before the 256 # first iteration anyway. 257 if self.bm_reboot and not self.bm_iterations_reboot: 258 self.reboot_target() 259 260 self.iterations_count = 1 261 262 def _preRun(self): 263 """ 264 Code executed before every iteration of the benchmark 265 """ 266 rebooted = False 267 268 if self.bm_reboot and self.bm_iterations_reboot: 269 rebooted = self.reboot_target() 270 271 if not rebooted and self.iterations_count > 1: 272 self._log.info('Waiting {}[s] before executing iteration {}...'\ 273 .format(self.bm_iterations_pause, self.iterations_count)) 274 sleep(self.bm_iterations_pause) 275 276 self.iterations_count += 1 277 278 def __init__(self): 279 """ 280 Set up logging and trigger running experiments 281 """ 282 LisaLogging.setup() 283 self._log = logging.getLogger('Benchmark') 284 285 self._log.info('=== CommandLine parsing...') 286 self.args = self._parseCommandLine() 287 288 self._log.info('=== TestEnv setup...') 289 self.bm_conf = self._getBmConf() 290 self.te = TestEnv(self.bm_conf) 291 self.target = self.te.target 292 293 self._log.info('=== Initialization...') 294 self.wl = self._getWorkload() 295 self.out_dir=self.te.res_dir 296 try: 297 self._preInit() 298 self.benchmarkInit() 299 except: 300 self._log.warning('Benchmark initialization failed: execution aborted') 301 raise 302 303 self._log.info('=== Execution...') 304 for iter_id in range(1, self.bm_iterations+1): 305 self._log.info('=== Iteration {}/{}...'.format(iter_id, self.bm_iterations)) 306 out_dir = os.path.join(self.out_dir, "{:03d}".format(iter_id)) 307 try: 308 os.makedirs(out_dir) 309 except: pass 310 311 self._preRun() 312 313 self.wl.run(out_dir=out_dir, 314 collect=self._getBmCollect(), 315 **self.bm_params) 316 317 self._log.info('=== Finalization...') 318 self.benchmarkFinalize() 319 320 def _wait_for_logcat_idle(self, seconds=1): 321 lines = 0 322 323 # Clear logcat 324 # os.system('{} logcat -s {} -c'.format(adb, DEVICE)); 325 self.target.clear_logcat() 326 327 # Dump logcat output 328 logcat_cmd = 'adb -s {} logcat'.format(self.target.adb_name) 329 logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) 330 logcat_poll = select.poll() 331 logcat_poll.register(logcat.stdout, select.POLLIN) 332 333 # Monitor logcat until it's idle for the specified number of [s] 334 self._log.info('Waiting for system to be almost idle') 335 self._log.info(' i.e. at least %d[s] of no logcat messages', seconds) 336 while True: 337 poll_result = logcat_poll.poll(seconds * 1000) 338 if not poll_result: 339 break 340 lines = lines + 1 341 line = logcat.stdout.readline(1024) 342 if lines % 1000: 343 self._log.debug(' still waiting...') 344 if lines > 1e6: 345 self._log.warning('device logcat seems quite busy, ' 346 'continuing anyway... ') 347 break 348 349 def reboot_target(self, disable_charge=True): 350 """ 351 Reboot the target if a "boot-image" has been specified 352 353 If the user specify a boot-image as a command line parameter, this 354 method will reboot the target with the specified kernel and wait 355 for the target to be up and running. 356 """ 357 rebooted = False 358 359 # Reboot the device, if a boot_image has been specified 360 if self.args.boot_image: 361 362 self._log.warning('=== Rebooting...') 363 self._log.warning('Rebooting image to use: %s', self.args.boot_image) 364 365 self._log.debug('Waiting 6[s] to enter bootloader...') 366 self.target.adb_reboot_bootloader() 367 sleep(6) 368 # self._fastboot('boot {}'.format(self.args.boot_image)) 369 cmd = 'boot {}'.format(self.args.boot_image) 370 fastboot_command(cmd, device=self.target.adb_name) 371 self._log.debug('Waiting {}[s] for boot to start...'\ 372 .format(self.args.boot_timeout)) 373 sleep(self.args.boot_timeout) 374 rebooted = True 375 376 else: 377 self._log.warning('Device NOT rebooted, using current image') 378 379 # Restart ADB in root mode 380 self._log.warning('Restarting ADB in root mode...') 381 self.target.adb_root(force=True) 382 383 # TODO add check for kernel SHA1 384 self._log.warning('Skipping kernel SHA1 cross-check...') 385 386 # Disable charge via USB 387 if disable_charge: 388 self._log.debug('Disabling charge over USB...') 389 self.target.charging_enabled = False 390 391 # Log current kernel version 392 self._log.info('Running with kernel:') 393 self._log.info(' %s', self.target.kernel_version) 394 395 # Wait for the system to complete the boot 396 self._wait_for_logcat_idle() 397 398 return rebooted 399 400# vim :set tabstop=4 shiftwidth=4 expandtab 401