1# -*- coding: utf-8 -*- 2# 3# Copyright 2020 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6""" 7Utils for Crosperf to setup devices. 8 9This script provides utils to set device specs. It is only used by Crosperf 10to setup device before running tests. 11 12""" 13 14from __future__ import division 15 16import logging 17import re 18import time 19 20from contextlib import contextmanager 21 22 23def run_command_on_dut(dut, command, ignore_status=False): 24 """ 25 Helper function to run command on DUT. 26 27 @param dut: The autotest host object representing DUT. 28 @param command: The command to run on DUT. 29 @ignore_status: Whether to ignore failure executing command. 30 31 @returns Return code, stdout, and stderr. 32 33 """ 34 result = dut.run(command, ignore_status=ignore_status) 35 36 ret, msg, err_msg = result.exit_status, result.stdout, result.stderr 37 38 if ret: 39 err_msg = ('Command execution on DUT %s failed.\n' 40 'Failing command: %s\n' 41 'returned %d\n' 42 'Error message: %s' % (dut.hostname, command, ret, err_msg)) 43 if ignore_status: 44 logging.warning(err_msg + 45 '\n(Failure is considered non-fatal. Continue.)') 46 else: 47 logging.error(err_msg) 48 49 return ret, msg, err_msg 50 51 52def disable_aslr(dut): 53 """ 54 Disable ASLR on DUT. 55 56 @param dut: The autotest host object representing DUT. 57 58 """ 59 disable_aslr = ('set -e; ' 60 'if [[ -e /proc/sys/kernel/randomize_va_space ]]; then ' 61 ' echo 0 > /proc/sys/kernel/randomize_va_space; ' 62 'fi') 63 logging.info('Disable ASLR.') 64 run_command_on_dut(dut, disable_aslr) 65 66 67def set_cpu_governor(dut, governor, ignore_status=False): 68 """ 69 Setup CPU Governor on DUT. 70 71 @param dut: The autotest host object representing DUT. 72 @param governor: CPU governor for DUT. 73 @ignore_status: Whether to ignore failure executing command. 74 75 @returns Return code of the command. 76 77 """ 78 set_gov_cmd = ( 79 'for f in `ls -d /sys/devices/system/cpu/cpu*/cpufreq 2>/dev/null`; do ' 80 # Skip writing scaling_governor if cpu is offline. 81 ' [[ -e ${f/cpufreq/online} ]] && grep -q 0 ${f/cpufreq/online} ' 82 ' && continue; ' 83 ' cd $f; ' 84 ' if [[ -e scaling_governor ]]; then ' 85 ' echo %s > scaling_governor; fi; ' 86 'done; ') 87 logging.info('Setup CPU Governor: %s.', governor) 88 ret, _, _ = run_command_on_dut( 89 dut, set_gov_cmd % governor, ignore_status=ignore_status) 90 return ret 91 92 93def disable_turbo(dut): 94 """ 95 Disable Turbo and Boost on DUT. 96 97 @param dut: The autotest host object representing DUT. 98 99 """ 100 dis_turbo_boost_cmd = ( 101 # Disable Turbo in the Intel pstate driver. 102 'if [[ -e /sys/devices/system/cpu/intel_pstate/no_turbo ]]; then ' 103 ' if grep -q 0 /sys/devices/system/cpu/intel_pstate/no_turbo; then ' 104 ' echo -n 1 > /sys/devices/system/cpu/intel_pstate/no_turbo; ' 105 ' fi; ' 106 'fi; ' 107 # Disable Boost on AMD. 108 'if [[ -e /sys/devices/system/cpu/cpufreq/boost ]]; then ' 109 ' if grep -q 1 /sys/devices/system/cpu/cpufreq/boost; then ' 110 ' echo -n 0 > /sys/devices/system/cpu/cpufreq/boost; ' 111 ' fi; ' 112 'fi; ' 113 ) 114 logging.info('Disable Turbo/Boost.') 115 run_command_on_dut(dut, dis_turbo_boost_cmd) 116 117 118def setup_cpu_usage(dut, cpu_usage): 119 """ 120 Setup CPU usage. 121 122 Based on dut_config['cpu_usage'] configure CPU cores utilization. 123 124 @param dut: The autotest host object representing DUT. 125 @param cpu_usage: Big/little core usage for CPU. 126 127 """ 128 129 if cpu_usage in ('big_only', 'little_only'): 130 _, arch, _ = run_command_on_dut(dut, 'uname -m') 131 132 if arch.lower().startswith('arm') or arch.lower().startswith('aarch64'): 133 setup_arm_cores(dut, cpu_usage) 134 135 136def setup_arm_cores(dut, cpu_usage): 137 """ 138 Setup ARM big/little cores. 139 140 @param dut: The autotest host object representing DUT. 141 @param cpu_usage: Big/little core usage for CPU. 142 143 """ 144 145 # CPU implemeters/part numbers of big/LITTLE CPU. 146 # Format: dict(CPU implementer: set(CPU part numbers)) 147 LITTLE_CORES = { 148 '0x41': { 149 '0xd01', # Cortex A32 150 '0xd03', # Cortex A53 151 '0xd04', # Cortex A35 152 '0xd05', # Cortex A55 153 }, 154 } 155 BIG_CORES = { 156 '0x41': { 157 '0xd07', # Cortex A57 158 '0xd08', # Cortex A72 159 '0xd09', # Cortex A73 160 '0xd0a', # Cortex A75 161 '0xd0b', # Cortex A76 162 }, 163 } 164 165 # Values of CPU Implementer and CPU part number are exposed by cpuinfo. 166 # Format: 167 # ================= 168 # processor : 0 169 # model name : ARMv8 Processor rev 4 (v8l) 170 # BogoMIPS : 48.00 171 # Features : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 172 # CPU implementer : 0x41 173 # CPU architecture: 8 174 # CPU variant : 0x0 175 # CPU part : 0xd03 176 # CPU revision : 4 177 178 _, cpuinfo, _ = run_command_on_dut(dut, 'cat /proc/cpuinfo') 179 180 # List of all CPU cores: 0, 1, .. 181 proc_matches = re.findall(r'^processor\s*: (\d+)$', cpuinfo, re.MULTILINE) 182 # List of all corresponding CPU implementers 183 impl_matches = re.findall(r'^CPU implementer\s*: (0x[\da-f]+)$', cpuinfo, 184 re.MULTILINE) 185 # List of all corresponding CPU part numbers 186 part_matches = re.findall(r'^CPU part\s*: (0x[\da-f]+)$', cpuinfo, 187 re.MULTILINE) 188 assert len(proc_matches) == len(impl_matches) 189 assert len(part_matches) == len(impl_matches) 190 191 all_cores = set(proc_matches) 192 dut_big_cores = { 193 core 194 for core, impl, part in zip(proc_matches, impl_matches, part_matches) 195 if impl in BIG_CORES and part in BIG_CORES[impl] 196 } 197 dut_lit_cores = { 198 core 199 for core, impl, part in zip(proc_matches, impl_matches, part_matches) 200 if impl in LITTLE_CORES and part in LITTLE_CORES[impl] 201 } 202 203 if cpu_usage == 'big_only': 204 cores_to_enable = dut_big_cores 205 cores_to_disable = all_cores - dut_big_cores 206 elif cpu_usage == 'little_only': 207 cores_to_enable = dut_lit_cores 208 cores_to_disable = all_cores - dut_lit_cores 209 else: 210 logging.warning( 211 'cpu_usage=%s is not supported on ARM.\n' 212 'Ignore ARM CPU setup and continue.', cpu_usage) 213 return 214 215 if cores_to_enable: 216 cmd_enable_cores = ( 217 'echo 1 | tee /sys/devices/system/cpu/cpu{%s}/online' % 218 ','.join(sorted(cores_to_enable))) 219 220 cmd_disable_cores = '' 221 if cores_to_disable: 222 cmd_disable_cores = ( 223 'echo 0 | tee /sys/devices/system/cpu/cpu{%s}/online' % 224 ','.join(sorted(cores_to_disable))) 225 226 run_command_on_dut(dut, '; '.join([cmd_enable_cores, 227 cmd_disable_cores])) 228 else: 229 # If there are no cores enabled by dut_config then configuration 230 # is invalid for current platform and should be ignored. 231 logging.warning( 232 '"cpu_usage" is invalid for targeted platform.\n' 233 'dut_config[cpu_usage]=%s\n' 234 'dut big cores: %s\n' 235 'dut little cores: %s\n' 236 'Ignore ARM CPU setup and continue.', 237 cpu_usage, dut_big_cores, dut_lit_cores) 238 239 240def get_cpu_online(dut): 241 """ 242 Get online status of CPU cores. 243 244 @param dut: The autotest host object representing DUT. 245 246 @returns dict of {int(cpu_num): <0|1>}. 247 248 """ 249 get_cpu_online_cmd = ('paste -d" "' 250 ' <(ls /sys/devices/system/cpu/cpu*/online)' 251 ' <(cat /sys/devices/system/cpu/cpu*/online)') 252 _, online_output_str, _ = run_command_on_dut(dut, get_cpu_online_cmd) 253 254 # Here is the output we expect to see: 255 # ----------------- 256 # /sys/devices/system/cpu/cpu0/online 0 257 # /sys/devices/system/cpu/cpu1/online 1 258 259 cpu_online = {} 260 cpu_online_match = re.compile(r'^[/\S]+/cpu(\d+)/[/\S]+\s+(\d+)$') 261 for line in online_output_str.splitlines(): 262 match = cpu_online_match.match(line) 263 if match: 264 cpu = int(match.group(1)) 265 status = int(match.group(2)) 266 cpu_online[cpu] = status 267 # There are platforms where CPU0 can't be disables and 268 # corresponding online file is not exposed. 269 # We need to add core 0 if it is present in the list of all online 270 # CPU cores. 271 get_all_online_cmd = 'cat /sys/devices/system/cpu/online' 272 _, all_online_str, _ = run_command_on_dut(dut, get_all_online_cmd) 273 # If cpu0 not in the online list and it exists in all online CPUs 274 # add it to the online list. 275 if 0 not in cpu_online and '0' in all_online_str: 276 # Add core0 to online cores. 277 cpu_online[0] = 1 278 # At least one CPU has to be online. 279 assert cpu_online 280 281 return cpu_online 282 283 284def setup_cpu_freq(dut, freq_percent, online_cores): 285 """Setup CPU frequency. 286 287 Based on dut_config['cpu_freq_pct'] setup frequency of online CPU cores 288 to a supported value which is less or equal to (freq_pct * max_freq / 100) 289 limited by min_freq. 290 291 NOTE: scaling_available_frequencies support is required. 292 Otherwise the function has no effect. 293 294 @param dut: The autotest host object representing DUT. 295 @param freq_percent: Frequency of online CPU cores to set. 296 @param online_cores: List of online cores (non-empty). 297 298 """ 299 if len(online_cores) == 1: 300 cpu_list_shell_str = str(online_cores[0]) 301 else: 302 cpu_list_shell_str = '{' + ','.join(str(core) 303 for core in online_cores) + '}' 304 list_all_avail_freq_cmd = ('ls /sys/devices/system/cpu/cpu' + 305 cpu_list_shell_str + 306 '/cpufreq/scaling_available_frequencies') 307 # Ignore error to support general usage of frequency setup. 308 # Not all platforms support scaling_available_frequencies. 309 ret, all_avail_freq_str, _ = run_command_on_dut(dut, 310 list_all_avail_freq_cmd, 311 ignore_status=True) 312 if ret or not all_avail_freq_str: 313 # No scalable frequencies available for the core. 314 return ret 315 for avail_freq_path in all_avail_freq_str.split(): 316 # Get available freq from every scaling_available_frequency path. 317 # Error is considered fatal in run_command_on_dut(). 318 _, avail_freq_str, _ = run_command_on_dut(dut, 'cat ' + avail_freq_path) 319 assert avail_freq_str 320 321 all_avail_freq = sorted( 322 int(freq_str) for freq_str in avail_freq_str.split()) 323 min_freq = all_avail_freq[0] 324 max_freq = all_avail_freq[-1] 325 # Calculate the frequency we are targeting. 326 target_freq = round(max_freq * freq_percent / 100) 327 # More likely it's not in the list of supported frequencies 328 # and our goal is to find the one which is less or equal. 329 # Default is min and we will try to maximize it. 330 avail_ngt_target = min_freq 331 # Find the largest not greater than the target. 332 for next_largest in reversed(all_avail_freq): 333 if next_largest <= target_freq: 334 avail_ngt_target = next_largest 335 break 336 337 max_freq_path = avail_freq_path.replace('scaling_available_frequencies', 338 'scaling_max_freq') 339 min_freq_path = avail_freq_path.replace('scaling_available_frequencies', 340 'scaling_min_freq') 341 # With default ignore_status=False we expect 0 status or Fatal error. 342 run_command_on_dut( 343 dut, 'echo %s | tee %s %s' % 344 (avail_ngt_target, max_freq_path, min_freq_path)) 345 346 347def wait_cooldown(dut, cooldown_time, cooldown_temp): 348 """ 349 Wait for DUT to cool down to certain temperature. 350 351 @param cooldown_time: Cooldown timeout. 352 @param cooldown_temp: Temperature to cooldown to. 353 354 @returns Cooldown wait time. 355 356 """ 357 waittime = 0 358 timeout_in_sec = int(cooldown_time) * 60 359 # Temperature from sensors come in uCelsius units. 360 temp_in_ucels = int(cooldown_temp) * 1000 361 sleep_interval = 30 362 363 # Wait until any of two events occurs: 364 # 1. CPU cools down to a specified temperature. 365 # 2. Timeout cooldown_time expires. 366 # For the case when targeted temperature is not reached within specified 367 # timeout the benchmark is going to start with higher initial CPU temp. 368 # In the worst case it may affect test results but at the same time we 369 # guarantee the upper bound of waiting time. 370 # TODO(denik): Report (or highlight) "high" CPU temperature in test results. 371 # "high" should be calculated based on empirical data per platform. 372 # Based on such reports we can adjust CPU configuration or 373 # cooldown limits accordingly. 374 while waittime < timeout_in_sec: 375 _, temp_output, _ = run_command_on_dut( 376 dut, 377 'cat /sys/class/thermal/thermal_zone*/temp', 378 ignore_status=True) 379 if any(int(temp) > temp_in_ucels for temp in temp_output.split()): 380 time.sleep(sleep_interval) 381 waittime += sleep_interval 382 else: 383 # Exit the loop when: 384 # 1. Reported temp numbers from all thermal sensors do not exceed 385 # 'cooldown_temp' or 386 # 2. No data from the sensors. 387 break 388 389 logging.info('Cooldown wait time: %.1f min', (waittime / 60)) 390 return waittime 391 392 393def decrease_wait_time(dut): 394 """ 395 Change the ten seconds wait time for pagecycler to two seconds. 396 397 @param dut: The autotest host object representing DUT. 398 399 """ 400 FILE = '/usr/local/telemetry/src/tools/perf/page_sets/page_cycler_story.py' 401 ret = run_command_on_dut(dut, 'ls ' + FILE) 402 403 if not ret: 404 sed_command = 'sed -i "s/_TTI_WAIT_TIME = 10/_TTI_WAIT_TIME = 2/g" ' 405 run_command_on_dut(dut, sed_command + FILE) 406 407 408def stop_ui(dut): 409 """ 410 Stop UI on DUT. 411 412 @param dut: The autotest host object representing DUT. 413 414 """ 415 run_command_on_dut(dut, 'stop ui') 416 417 418def start_ui(dut): 419 """ 420 Start UI on DUT. 421 422 @param dut: The autotest host object representing DUT. 423 424 """ 425 run_command_on_dut(dut, 'start ui') 426 427 428def kern_cmd_update_needed(dut, intel_pstate): 429 """ 430 Check whether kernel cmdline update is needed. 431 432 @param dut: The autotest host object representing DUT. 433 @param intel_pstate: kernel command line argument (active, passive, 434 no_hwp). 435 436 @returns True if update is needed. 437 438 """ 439 good = 0 440 441 # Check that dut platform supports hwp 442 cmd = "grep -q '^flags.*hwp' /proc/cpuinfo" 443 ret_code, _, _ = run_command_on_dut(dut, cmd, ignore_status=True) 444 if ret_code != good: 445 # Intel hwp is not supported, update is not needed. 446 return False 447 448 kern_cmdline_cmd = 'grep -q "intel_pstate=%s" /proc/cmdline' % intel_pstate 449 ret_code, _, _ = run_command_on_dut( 450 dut, kern_cmdline_cmd, ignore_status=True) 451 logging.info('grep /proc/cmdline returned %d', ret_code) 452 if (intel_pstate and ret_code == good or 453 not intel_pstate and ret_code != good): 454 # No need to updated cmdline if: 455 # 1. We are setting intel_pstate and we found it is already set. 456 # 2. Not using intel_pstate and it is not in cmdline. 457 return False 458 459 # Otherwise we need to update intel_pstate. 460 return True 461 462 463def update_kern_cmd_intel_pstate(dut, intel_pstate): 464 """Update kernel command line. 465 466 @param dut: The autotest host object representing DUT. 467 @param intel_pstate: kernel command line argument(active, passive, 468 no_hwp). 469 470 """ 471 good = 0 472 473 # First phase is to remove rootfs verification to allow cmdline change. 474 remove_verif_cmd = ' '.join([ 475 '/usr/share/vboot/bin/make_dev_ssd.sh', 476 '--remove_rootfs_verification', 477 '--partition %d', 478 ]) 479 # Command for partition 2. 480 verif_part2_failed, _, _ = run_command_on_dut( 481 dut, remove_verif_cmd % 2, ignore_status=True) 482 # Command for partition 4 483 # Some machines in the lab use partition 4 to boot from, 484 # so cmdline should be update for both partitions. 485 verif_part4_failed, _, _ = run_command_on_dut( 486 dut, remove_verif_cmd % 4, ignore_status=True) 487 if verif_part2_failed or verif_part4_failed: 488 logging.error( 489 'ERROR. Failed to update kernel cmdline on partition %d.\n' 490 'Remove verification failed with status %d', 491 2 if verif_part2_failed else 4, verif_part2_failed or 492 verif_part4_failed) 493 494 run_command_on_dut(dut, 'reboot && exit') 495 # Give enough time for dut to complete reboot 496 # TODO(denik): Replace with the function checking machine availability. 497 time.sleep(30) 498 499 # Second phase to update intel_pstate in kernel cmdline. 500 kern_cmdline = '\n'.join([ 501 'tmpfile=$(mktemp)', 502 'partnumb=%d', 503 'pstate=%s', 504 # Store kernel cmdline in a temp file. 505 '/usr/share/vboot/bin/make_dev_ssd.sh --partition ${partnumb}' 506 ' --save_config ${tmpfile}', 507 # Remove intel_pstate argument if present. 508 "sed -i -r 's/ intel_pstate=[A-Za-z_]+//g' ${tmpfile}.${partnumb}", 509 # Insert intel_pstate with a new value if it is set. 510 '[[ -n ${pstate} ]] &&' 511 ' sed -i -e \"s/ *$/ intel_pstate=${pstate}/\" ${tmpfile}.${partnumb}', 512 # Save the change in kernel cmdline. 513 # After completion we have to reboot. 514 '/usr/share/vboot/bin/make_dev_ssd.sh --partition ${partnumb}' 515 ' --set_config ${tmpfile}' 516 ]) 517 kern_part2_cmdline_cmd = kern_cmdline % (2, intel_pstate) 518 logging.info('Command to change kernel command line: %s', 519 kern_part2_cmdline_cmd) 520 upd_part2_failed, _, _ = run_command_on_dut( 521 dut, kern_part2_cmdline_cmd, ignore_status=True) 522 # Again here we are updating cmdline for partition 4 523 # in addition to partition 2. Without this some machines 524 # in the lab might fail. 525 kern_part4_cmdline_cmd = kern_cmdline % (4, intel_pstate) 526 logging.info('Command to change kernel command line: %s', 527 kern_part4_cmdline_cmd) 528 upd_part4_failed, _, _ = run_command_on_dut( 529 dut, kern_part4_cmdline_cmd, ignore_status=True) 530 if upd_part2_failed or upd_part4_failed: 531 logging.error( 532 'ERROR. Failed to update kernel cmdline on partition %d.\n' 533 'intel_pstate update failed with status %d', 534 2 if upd_part2_failed else 4, upd_part2_failed or upd_part4_failed) 535 536 run_command_on_dut(dut, 'reboot && exit') 537 # Wait 30s after reboot. 538 time.sleep(30) 539 540 # Verification phase. 541 # Check that cmdline was updated. 542 # Throw an exception if not. 543 kern_cmdline_cmd = 'grep -q "intel_pstate=%s" /proc/cmdline' % intel_pstate 544 ret_code, _, _ = run_command_on_dut( 545 dut, kern_cmdline_cmd, ignore_status=True) 546 if (intel_pstate and ret_code != good or 547 not intel_pstate and ret_code == good): 548 # Kernel cmdline doesn't match input intel_pstate. 549 logging.error( 550 'ERROR. Failed to update kernel cmdline. ' 551 'Final verification failed with status %d', ret_code) 552 553 logging.info('Kernel cmdline updated successfully.') 554 555 556@contextmanager 557def pause_ui(dut): 558 """ 559 Stop UI before and Start UI after the context block. 560 561 Context manager will make sure UI is always resumed at the end. 562 563 @param dut: The autotest host object representing DUT. 564 565 """ 566 stop_ui(dut) 567 try: 568 yield 569 570 finally: 571 start_ui(dut) 572 573 574def setup_device(dut, dut_config): 575 """ 576 Setup device to get it ready for testing. 577 578 @param dut: The autotest host object representing DUT. 579 @param dut_config: A dictionary of DUT configurations. 580 581 @returns Wait time of cool down for this benchmark run. 582 583 """ 584 logging.info('Update kernel cmdline if necessary and reboot') 585 intel_pstate = dut_config.get('intel_pstate') 586 if intel_pstate and kern_cmd_update_needed(dut, intel_pstate): 587 update_kern_cmd_intel_pstate(dut, intel_pstate) 588 589 wait_time = 0 590 # Pause UI while configuring the DUT. 591 # This will accelerate setup (waiting for cooldown has x10 drop) 592 # and help to reset a Chrome state left after the previous test. 593 with pause_ui(dut): 594 # Unless the user turns on ASLR in the flag, we first disable ASLR 595 # before running the benchmarks 596 if not dut_config.get('enable_aslr'): 597 disable_aslr(dut) 598 599 # CPU usage setup comes first where we enable/disable cores. 600 setup_cpu_usage(dut, dut_config.get('cpu_usage')) 601 cpu_online_status = get_cpu_online(dut) 602 # List of online cores of type int (core number). 603 online_cores = [ 604 core for core, status in cpu_online_status.items() if status 605 ] 606 if dut_config.get('cooldown_time'): 607 # Setup power conservative mode for effective cool down. 608 # Set ignore status since powersave may no be available 609 # on all platforms and we are going to handle it. 610 ret = set_cpu_governor(dut, 'powersave', ignore_status=True) 611 if ret: 612 # "powersave" is not available, use "ondemand". 613 # Still not a fatal error if it fails. 614 ret = set_cpu_governor(dut, 'ondemand', ignore_status=True) 615 # TODO(denik): Run comparison test for 'powersave' and 'ondemand' 616 # on scarlet and kevin64. 617 # We might have to consider reducing freq manually to the min 618 # if it helps to reduce waiting time. 619 wait_time = wait_cooldown( 620 dut, dut_config['cooldown_time'], dut_config['cooldown_temp']) 621 622 # Setup CPU governor for the benchmark run. 623 # It overwrites the previous governor settings. 624 governor = dut_config.get('governor') 625 # FIXME(denik): Pass online cores to governor setup. 626 if governor: 627 set_cpu_governor(dut, governor) 628 629 # Disable Turbo and Setup CPU freq should ALWAYS proceed governor setup 630 # since governor may change: 631 # - frequency; 632 # - turbo/boost. 633 disable_turbo(dut) 634 if dut_config.get('cpu_freq_pct'): 635 setup_cpu_freq(dut, dut_config['cpu_freq_pct'], online_cores) 636 637 decrease_wait_time(dut) 638 # FIXME(denik): Currently we are not recovering the previous cpufreq 639 # settings since we do reboot/setup every time anyway. 640 # But it may change in the future and then we have to recover the 641 # settings. 642 return wait_time 643