• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2#
3# Copyright 2019 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Utils for setting devices
8
9This script provides utils to set device specs.
10"""
11
12from __future__ import division
13from __future__ import print_function
14
15__author__ = 'zhizhouy@google.com (Zhizhou Yang)'
16
17import re
18import time
19
20from contextlib import contextmanager
21
22from cros_utils import command_executer
23
24
25class DutWrapper(object):
26  """Wrap DUT parameters inside."""
27
28  def __init__(self,
29               chromeos_root,
30               remote,
31               log_level='verbose',
32               logger=None,
33               ce=None,
34               dut_config=None):
35    self.chromeos_root = chromeos_root
36    self.remote = remote
37    self.log_level = log_level
38    self.logger = logger
39    self.ce = ce or command_executer.GetCommandExecuter(log_level=log_level)
40    self.dut_config = dut_config
41
42  def RunCommandOnDut(self, command, ignore_status=False):
43    """Helper function to run command on DUT."""
44    ret, msg, err_msg = self.ce.CrosRunCommandWOutput(
45        command, machine=self.remote, chromeos_root=self.chromeos_root)
46
47    if ret:
48      err_msg = ('Command execution on DUT %s failed.\n'
49                 'Failing command: %s\n'
50                 'returned %d\n'
51                 'Error message: %s' % (self.remote, command, ret, err_msg))
52      if ignore_status:
53        self.logger.LogError(err_msg +
54                             '\n(Failure is considered non-fatal. Continue.)')
55      else:
56        self.logger.LogFatal(err_msg)
57
58    return ret, msg, err_msg
59
60  def DisableASLR(self):
61    """Disable ASLR on DUT."""
62    disable_aslr = ('set -e; '
63                    'if [[ -e /proc/sys/kernel/randomize_va_space ]]; then '
64                    '  echo 0 > /proc/sys/kernel/randomize_va_space; '
65                    'fi')
66    if self.log_level == 'average':
67      self.logger.LogOutput('Disable ASLR.')
68    self.RunCommandOnDut(disable_aslr)
69
70  def SetCpuGovernor(self, governor, ignore_status=False):
71    """Setup CPU Governor on DUT."""
72    set_gov_cmd = (
73        'for f in `ls -d /sys/devices/system/cpu/cpu*/cpufreq 2>/dev/null`; do '
74        # Skip writing scaling_governor if cpu is offline.
75        ' [[ -e ${f/cpufreq/online} ]] && grep -q 0 ${f/cpufreq/online} '
76        '   && continue; '
77        ' cd $f; '
78        ' if [[ -e scaling_governor ]]; then '
79        '  echo %s > scaling_governor; fi; '
80        'done; ')
81    if self.log_level == 'average':
82      self.logger.LogOutput('Setup CPU Governor: %s.' % governor)
83    ret, _, _ = self.RunCommandOnDut(
84        set_gov_cmd % governor, ignore_status=ignore_status)
85    return ret
86
87  def DisableTurbo(self):
88    """Disable Turbo on DUT."""
89    dis_turbo_cmd = (
90        'if [[ -e /sys/devices/system/cpu/intel_pstate/no_turbo ]]; then '
91        '  if grep -q 0 /sys/devices/system/cpu/intel_pstate/no_turbo;  then '
92        '    echo -n 1 > /sys/devices/system/cpu/intel_pstate/no_turbo; '
93        '  fi; '
94        'fi; ')
95    if self.log_level == 'average':
96      self.logger.LogOutput('Disable Turbo.')
97    self.RunCommandOnDut(dis_turbo_cmd)
98
99  def SetupCpuUsage(self):
100    """Setup CPU usage.
101
102    Based on self.dut_config['cpu_usage'] configure CPU cores
103    utilization.
104    """
105
106    if (self.dut_config['cpu_usage'] == 'big_only' or
107        self.dut_config['cpu_usage'] == 'little_only'):
108      _, arch, _ = self.RunCommandOnDut('uname -m')
109
110      if arch.lower().startswith('arm') or arch.lower().startswith('aarch64'):
111        self.SetupArmCores()
112
113  def SetupArmCores(self):
114    """Setup ARM big/little cores."""
115
116    # CPU implemeters/part numbers of big/LITTLE CPU.
117    # Format: dict(CPU implementer: set(CPU part numbers))
118    LITTLE_CORES = {
119        '0x41': {
120            '0xd01',  # Cortex A32
121            '0xd03',  # Cortex A53
122            '0xd04',  # Cortex A35
123            '0xd05',  # Cortex A55
124        },
125    }
126    BIG_CORES = {
127        '0x41': {
128            '0xd07',  # Cortex A57
129            '0xd08',  # Cortex A72
130            '0xd09',  # Cortex A73
131            '0xd0a',  # Cortex A75
132            '0xd0b',  # Cortex A76
133        },
134    }
135
136    # Values of CPU Implementer and CPU part number are exposed by cpuinfo.
137    # Format:
138    # =================
139    # processor       : 0
140    # model name      : ARMv8 Processor rev 4 (v8l)
141    # BogoMIPS        : 48.00
142    # Features        : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4
143    # CPU implementer : 0x41
144    # CPU architecture: 8
145    # CPU variant     : 0x0
146    # CPU part        : 0xd03
147    # CPU revision    : 4
148
149    _, cpuinfo, _ = self.RunCommandOnDut('cat /proc/cpuinfo')
150
151    # List of all CPU cores: 0, 1, ..
152    proc_matches = re.findall(r'^processor\s*: (\d+)$', cpuinfo, re.MULTILINE)
153    # List of all corresponding CPU implementers
154    impl_matches = re.findall(r'^CPU implementer\s*: (0x[\da-f]+)$', cpuinfo,
155                              re.MULTILINE)
156    # List of all corresponding CPU part numbers
157    part_matches = re.findall(r'^CPU part\s*: (0x[\da-f]+)$', cpuinfo,
158                              re.MULTILINE)
159    assert len(proc_matches) == len(impl_matches)
160    assert len(part_matches) == len(impl_matches)
161
162    all_cores = set(proc_matches)
163    dut_big_cores = {
164        core
165        for core, impl, part in zip(proc_matches, impl_matches, part_matches)
166        if impl in BIG_CORES and part in BIG_CORES[impl]
167    }
168    dut_lit_cores = {
169        core
170        for core, impl, part in zip(proc_matches, impl_matches, part_matches)
171        if impl in LITTLE_CORES and part in LITTLE_CORES[impl]
172    }
173
174    if self.dut_config['cpu_usage'] == 'big_only':
175      cores_to_enable = dut_big_cores
176      cores_to_disable = all_cores - dut_big_cores
177    elif self.dut_config['cpu_usage'] == 'little_only':
178      cores_to_enable = dut_lit_cores
179      cores_to_disable = all_cores - dut_lit_cores
180    else:
181      self.logger.LogError(
182          'cpu_usage=%s is not supported on ARM.\n'
183          'Ignore ARM CPU setup and continue.' % self.dut_config['cpu_usage'])
184      return
185
186    if cores_to_enable:
187      cmd_enable_cores = ('echo 1 | tee /sys/devices/system/cpu/cpu{%s}/online'
188                          % ','.join(sorted(cores_to_enable)))
189
190      cmd_disable_cores = ''
191      if cores_to_disable:
192        cmd_disable_cores = (
193            'echo 0 | tee /sys/devices/system/cpu/cpu{%s}/online' % ','.join(
194                sorted(cores_to_disable)))
195
196      self.RunCommandOnDut('; '.join([cmd_enable_cores, cmd_disable_cores]))
197    else:
198      # If there are no cores enabled by dut_config then configuration
199      # is invalid for current platform and should be ignored.
200      self.logger.LogError(
201          '"cpu_usage" is invalid for targeted platform.\n'
202          'dut_config[cpu_usage]=%s\n'
203          'dut big cores: %s\n'
204          'dut little cores: %s\n'
205          'Ignore ARM CPU setup and continue.' % (self.dut_config['cpu_usage'],
206                                                  dut_big_cores, dut_lit_cores))
207
208  def GetCpuOnline(self):
209    """Get online status of CPU cores.
210
211    Return dict of {int(cpu_num): <0|1>}.
212    """
213    get_cpu_online_cmd = ('paste -d" "'
214                          ' <(ls /sys/devices/system/cpu/cpu*/online)'
215                          ' <(cat /sys/devices/system/cpu/cpu*/online)')
216    _, online_output_str, _ = self.RunCommandOnDut(get_cpu_online_cmd)
217
218    # Here is the output we expect to see:
219    # -----------------
220    # /sys/devices/system/cpu/cpu0/online 0
221    # /sys/devices/system/cpu/cpu1/online 1
222
223    cpu_online = {}
224    cpu_online_match = re.compile(r'^[/\S]+/cpu(\d+)/[/\S]+\s+(\d+)$')
225    for line in online_output_str.splitlines():
226      match = cpu_online_match.match(line)
227      if match:
228        cpu = int(match.group(1))
229        status = int(match.group(2))
230        cpu_online[cpu] = status
231    # At least one CPU has to be online.
232    assert cpu_online
233
234    return cpu_online
235
236  def SetupCpuFreq(self, online_cores):
237    """Setup CPU frequency.
238
239    Based on self.dut_config['cpu_freq_pct'] setup frequency of online CPU cores
240    to a supported value which is less or equal to (freq_pct * max_freq / 100)
241    limited by min_freq.
242
243    NOTE: scaling_available_frequencies support is required.
244    Otherwise the function has no effect.
245    """
246    freq_percent = self.dut_config['cpu_freq_pct']
247    list_all_avail_freq_cmd = ('ls /sys/devices/system/cpu/cpu{%s}/cpufreq/'
248                               'scaling_available_frequencies')
249    # Ignore error to support general usage of frequency setup.
250    # Not all platforms support scaling_available_frequencies.
251    ret, all_avail_freq_str, _ = self.RunCommandOnDut(
252        list_all_avail_freq_cmd % ','.join(str(core) for core in online_cores),
253        ignore_status=True)
254    if ret or not all_avail_freq_str:
255      # No scalable frequencies available for the core.
256      return ret
257    for avail_freq_path in all_avail_freq_str.split():
258      # Get available freq from every scaling_available_frequency path.
259      # Error is considered fatal in self.RunCommandOnDut().
260      _, avail_freq_str, _ = self.RunCommandOnDut('cat ' + avail_freq_path)
261      assert avail_freq_str
262
263      all_avail_freq = sorted(
264          int(freq_str) for freq_str in avail_freq_str.split())
265      min_freq = all_avail_freq[0]
266      max_freq = all_avail_freq[-1]
267      # Calculate the frequency we are targeting.
268      target_freq = round(max_freq * freq_percent / 100)
269      # More likely it's not in the list of supported frequencies
270      # and our goal is to find the one which is less or equal.
271      # Default is min and we will try to maximize it.
272      avail_ngt_target = min_freq
273      # Find the largest not greater than the target.
274      for next_largest in reversed(all_avail_freq):
275        if next_largest <= target_freq:
276          avail_ngt_target = next_largest
277          break
278
279      max_freq_path = avail_freq_path.replace('scaling_available_frequencies',
280                                              'scaling_max_freq')
281      min_freq_path = avail_freq_path.replace('scaling_available_frequencies',
282                                              'scaling_min_freq')
283      # With default ignore_status=False we expect 0 status or Fatal error.
284      self.RunCommandOnDut('echo %s | tee %s %s' %
285                           (avail_ngt_target, max_freq_path, min_freq_path))
286
287  def WaitCooldown(self):
288    """Wait for DUT to cool down to certain temperature."""
289    waittime = 0
290    timeout_in_sec = int(self.dut_config['cooldown_time']) * 60
291    # Temperature from sensors come in uCelsius units.
292    temp_in_ucels = int(self.dut_config['cooldown_temp']) * 1000
293    sleep_interval = 30
294
295    # Wait until any of two events occurs:
296    # 1. CPU cools down to a specified temperature.
297    # 2. Timeout cooldown_time expires.
298    # For the case when targeted temperature is not reached within specified
299    # timeout the benchmark is going to start with higher initial CPU temp.
300    # In the worst case it may affect test results but at the same time we
301    # guarantee the upper bound of waiting time.
302    # TODO(denik): Report (or highlight) "high" CPU temperature in test results.
303    # "high" should be calculated based on empirical data per platform.
304    # Based on such reports we can adjust CPU configuration or
305    # cooldown limits accordingly.
306    while waittime < timeout_in_sec:
307      _, temp_output, _ = self.RunCommandOnDut(
308          'cat /sys/class/thermal/thermal_zone*/temp', ignore_status=True)
309      if any(int(temp) > temp_in_ucels for temp in temp_output.split()):
310        time.sleep(sleep_interval)
311        waittime += sleep_interval
312      else:
313        # Exit the loop when:
314        # 1. Reported temp numbers from all thermal sensors do not exceed
315        # 'cooldown_temp' or
316        # 2. No data from the sensors.
317        break
318
319    self.logger.LogOutput('Cooldown wait time: %.1f min' % (waittime / 60))
320    return waittime
321
322  def DecreaseWaitTime(self):
323    """Change the ten seconds wait time for pagecycler to two seconds."""
324    FILE = '/usr/local/telemetry/src/tools/perf/page_sets/page_cycler_story.py'
325    ret = self.RunCommandOnDut('ls ' + FILE)
326
327    if not ret:
328      sed_command = 'sed -i "s/_TTI_WAIT_TIME = 10/_TTI_WAIT_TIME = 2/g" '
329      self.RunCommandOnDut(sed_command + FILE)
330
331  def StopUI(self):
332    """Stop UI on DUT."""
333    # Added "ignore_status" for the case when crosperf stops ui service which
334    # was already stopped. Command is going to fail with 1.
335    self.RunCommandOnDut('stop ui', ignore_status=True)
336
337  def StartUI(self):
338    """Start UI on DUT."""
339    # Similar to StopUI, `start ui` fails if the service is already started.
340    self.RunCommandOnDut('start ui', ignore_status=True)
341
342  def KerncmdUpdateNeeded(self, intel_pstate):
343    """Check whether kernel cmdline update is needed.
344
345    Args:
346      intel_pstate: kernel command line argument (active, passive, no_hwp)
347
348    Returns:
349      True if update is needed.
350    """
351
352    good = 0
353
354    # Check that dut platform supports hwp
355    cmd = "grep -q '^flags.*hwp' /proc/cpuinfo"
356    ret_code, _, _ = self.RunCommandOnDut(cmd, ignore_status=True)
357    if ret_code != good:
358      # Intel hwp is not supported, update is not needed.
359      return False
360
361    kern_cmdline_cmd = 'grep -q "intel_pstate=%s" /proc/cmdline' % intel_pstate
362    ret_code, _, _ = self.RunCommandOnDut(kern_cmdline_cmd, ignore_status=True)
363    self.logger.LogOutput('grep /proc/cmdline returned %d' % ret_code)
364    if (intel_pstate and ret_code == good or
365        not intel_pstate and ret_code != good):
366      # No need to updated cmdline if:
367      # 1. We are setting intel_pstate and we found it is already set.
368      # 2. Not using intel_pstate and it is not in cmdline.
369      return False
370
371    # Otherwise we need to update intel_pstate.
372    return True
373
374  def UpdateKerncmdIntelPstate(self, intel_pstate):
375    """Update kernel command line.
376
377    Args:
378      intel_pstate: kernel command line argument (active, passive, no_hwp)
379    """
380
381    good = 0
382
383    # First phase is to remove rootfs verification to allow cmdline change.
384    remove_verif_cmd = ' '.join([
385        '/usr/share/vboot/bin/make_dev_ssd.sh',
386        '--remove_rootfs_verification',
387        '--partition %d',
388    ])
389    # Command for partition 2.
390    verif_part2_failed, _, _ = self.RunCommandOnDut(
391        remove_verif_cmd % 2, ignore_status=True)
392    # Command for partition 4
393    # Some machines in the lab use partition 4 to boot from,
394    # so cmdline should be update for both partitions.
395    verif_part4_failed, _, _ = self.RunCommandOnDut(
396        remove_verif_cmd % 4, ignore_status=True)
397    if verif_part2_failed or verif_part4_failed:
398      self.logger.LogFatal(
399          'ERROR. Failed to update kernel cmdline on partition %d.\n'
400          'Remove verification failed with status %d' %
401          (2 if verif_part2_failed else 4, verif_part2_failed or
402           verif_part4_failed))
403
404    self.RunCommandOnDut('reboot && exit')
405    # Give enough time for dut to complete reboot
406    # TODO(denik): Replace with the function checking machine availability.
407    time.sleep(30)
408
409    # Second phase to update intel_pstate in kernel cmdline.
410    kern_cmdline = '\n'.join([
411        'tmpfile=$(mktemp)',
412        'partnumb=%d',
413        'pstate=%s',
414        # Store kernel cmdline in a temp file.
415        '/usr/share/vboot/bin/make_dev_ssd.sh --partition ${partnumb}'
416        ' --save_config ${tmpfile}',
417        # Remove intel_pstate argument if present.
418        "sed -i -r 's/ intel_pstate=[A-Za-z_]+//g' ${tmpfile}.${partnumb}",
419        # Insert intel_pstate with a new value if it is set.
420        '[[ -n ${pstate} ]] &&'
421        ' sed -i -e \"s/ *$/ intel_pstate=${pstate}/\" ${tmpfile}.${partnumb}',
422        # Save the change in kernel cmdline.
423        # After completion we have to reboot.
424        '/usr/share/vboot/bin/make_dev_ssd.sh --partition ${partnumb}'
425        ' --set_config ${tmpfile}'
426    ])
427    kern_part2_cmdline_cmd = kern_cmdline % (2, intel_pstate)
428    self.logger.LogOutput(
429        'Command to change kernel command line: %s' % kern_part2_cmdline_cmd)
430    upd_part2_failed, _, _ = self.RunCommandOnDut(
431        kern_part2_cmdline_cmd, ignore_status=True)
432    # Again here we are updating cmdline for partition 4
433    # in addition to partition 2. Without this some machines
434    # in the lab might fail.
435    kern_part4_cmdline_cmd = kern_cmdline % (4, intel_pstate)
436    self.logger.LogOutput(
437        'Command to change kernel command line: %s' % kern_part4_cmdline_cmd)
438    upd_part4_failed, _, _ = self.RunCommandOnDut(
439        kern_part4_cmdline_cmd, ignore_status=True)
440    if upd_part2_failed or upd_part4_failed:
441      self.logger.LogFatal(
442          'ERROR. Failed to update kernel cmdline on partition %d.\n'
443          'intel_pstate update failed with status %d' %
444          (2 if upd_part2_failed else 4, upd_part2_failed or upd_part4_failed))
445
446    self.RunCommandOnDut('reboot && exit')
447    # Wait 30s after reboot.
448    time.sleep(30)
449
450    # Verification phase.
451    # Check that cmdline was updated.
452    # Throw an exception if not.
453    kern_cmdline_cmd = 'grep -q "intel_pstate=%s" /proc/cmdline' % intel_pstate
454    ret_code, _, _ = self.RunCommandOnDut(kern_cmdline_cmd, ignore_status=True)
455    if (intel_pstate and ret_code != good or
456        not intel_pstate and ret_code == good):
457      # Kernel cmdline doesn't match input intel_pstate.
458      self.logger.LogFatal(
459          'ERROR. Failed to update kernel cmdline. '
460          'Final verification failed with status %d' % ret_code)
461
462    self.logger.LogOutput('Kernel cmdline updated successfully.')
463
464  @contextmanager
465  def PauseUI(self):
466    """Stop UI before and Start UI after the context block.
467
468    Context manager will make sure UI is always resumed at the end.
469    """
470    self.StopUI()
471    try:
472      yield
473
474    finally:
475      self.StartUI()
476
477  def SetupDevice(self):
478    """Setup device to get it ready for testing.
479
480    @Returns Wait time of cool down for this benchmark run.
481    """
482    self.logger.LogOutput('Update kernel cmdline if necessary and reboot')
483    intel_pstate = self.dut_config['intel_pstate']
484    if intel_pstate and self.KerncmdUpdateNeeded(intel_pstate):
485      self.UpdateKerncmdIntelPstate(intel_pstate)
486
487    wait_time = 0
488    # Pause UI while configuring the DUT.
489    # This will accelerate setup (waiting for cooldown has x10 drop)
490    # and help to reset a Chrome state left after the previous test.
491    with self.PauseUI():
492      # Unless the user turns on ASLR in the flag, we first disable ASLR
493      # before running the benchmarks
494      if not self.dut_config['enable_aslr']:
495        self.DisableASLR()
496
497      # CPU usage setup comes first where we enable/disable cores.
498      self.SetupCpuUsage()
499      cpu_online_status = self.GetCpuOnline()
500      # List of online cores of type int (core number).
501      online_cores = [
502          core for core, status in cpu_online_status.items() if status
503      ]
504      if self.dut_config['cooldown_time']:
505        # Setup power conservative mode for effective cool down.
506        # Set ignore status since powersave may no be available
507        # on all platforms and we are going to handle it.
508        ret = self.SetCpuGovernor('powersave', ignore_status=True)
509        if ret:
510          # "powersave" is not available, use "ondemand".
511          # Still not a fatal error if it fails.
512          ret = self.SetCpuGovernor('ondemand', ignore_status=True)
513        # TODO(denik): Run comparison test for 'powersave' and 'ondemand'
514        # on scarlet and kevin64.
515        # We might have to consider reducing freq manually to the min
516        # if it helps to reduce waiting time.
517        wait_time = self.WaitCooldown()
518
519      # Setup CPU governor for the benchmark run.
520      # It overwrites the previous governor settings.
521      governor = self.dut_config['governor']
522      # FIXME(denik): Pass online cores to governor setup.
523      self.SetCpuGovernor(governor)
524
525      # Disable Turbo and Setup CPU freq should ALWAYS proceed governor setup
526      # since governor may change:
527      # - frequency;
528      # - turbo/boost.
529      self.DisableTurbo()
530      self.SetupCpuFreq(online_cores)
531
532      self.DecreaseWaitTime()
533      # FIXME(denik): Currently we are not recovering the previous cpufreq
534      # settings since we do reboot/setup every time anyway.
535      # But it may change in the future and then we have to recover the
536      # settings.
537    return wait_time
538