1# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import time 8 9from autotest_lib.client.bin import test, utils 10from autotest_lib.client.common_lib import error 11from autotest_lib.client.common_lib import file_utils 12from autotest_lib.client.common_lib.cros import chrome 13from autotest_lib.client.cros import service_stopper 14from autotest_lib.client.cros.power import power_status, power_utils 15from autotest_lib.client.cros.video import histogram_verifier 16from autotest_lib.client.cros.video import constants 17from autotest_lib.client.cros.video import helper_logger 18 19 20EXTRA_BROWSER_ARGS = ['--use-fake-device-for-media-stream', 21 '--use-fake-ui-for-media-stream'] 22FAKE_FILE_ARG = '--use-file-for-fake-video-capture="%s"' 23DISABLE_ACCELERATED_VIDEO_DECODE_BROWSER_ARGS = [ 24 '--disable-accelerated-video-decode'] 25 26DOWNLOAD_BASE = ('http://commondatastorage.googleapis.com/' 27 'chromiumos-test-assets-public/crowd/') 28VIDEO_NAME = 'crowd720_25frames.y4m' 29 30WEBRTC_INTERNALS_URL = 'chrome://webrtc-internals' 31 32WEBRTC_WITH_HW_ACCELERATION = 'webrtc_with_hw_acceleration' 33WEBRTC_WITHOUT_HW_ACCELERATION = 'webrtc_without_hw_acceleration' 34 35# Measurement duration in seconds. 36MEASUREMENT_DURATION = 30 37# Time to exclude from calculation after start the loopback [seconds]. 38STABILIZATION_DURATION = 10 39 40# List of thermal throttling services that should be disabled. 41# - temp_metrics for link. 42# - thermal for daisy, snow, pit etc. 43THERMAL_SERVICES = ['temp_metrics', 'thermal'] 44 45# Time in seconds to wait for cpu idle until giving up. 46WAIT_FOR_IDLE_CPU_TIMEOUT = 60.0 47# Maximum percent of cpu usage considered as idle. 48CPU_IDLE_USAGE = 0.1 49 50MAX_DECODE_TIME_DESCRIPTION = 'decode_time.max' 51MEDIAN_DECODE_TIME_DESCRIPTION = 'decode_time.percentile_0.50' 52CPU_USAGE_DESCRIPTION = 'video_cpu_usage' 53POWER_DESCRIPTION = 'video_mean_energy_rate' 54 55# Minimum battery charge percentage to run the power test. 56BATTERY_INITIAL_CHARGED_MIN = 10 57 58# These are the variable names in WebRTC internals. 59# Maximum decode time of the frames of the last 10 seconds. 60GOOG_MAX_DECODE_MS = 'googMaxDecodeMs' 61# The decode time of the last frame. 62GOOG_DECODE_MS = 'googDecodeMs' 63 64# Javascript function to get the decode time. addStats is a function called by 65# Chrome to pass WebRTC statistics every second. 66ADD_STATS_JAVASCRIPT = ( 67 'var googMaxDecodeMs = new Array();' 68 'var googDecodeMs = new Array();' 69 'addStats = function(data) {' 70 ' reports = data.reports;' 71 ' for (var i=0; i < reports.length; i++) {' 72 ' if (reports[i].type == "ssrc") {' 73 ' values = reports[i].stats.values;' 74 ' for (var j=0; j < values.length; j++) {' 75 ' if (values[j] == "googMaxDecodeMs")' 76 ' googMaxDecodeMs[googMaxDecodeMs.length] = values[j+1];' 77 ' else if (values[j] == "googDecodeMs")' 78 ' googDecodeMs[googDecodeMs.length] = values[j+1];' 79 ' }' 80 ' }' 81 ' }' 82 '}') 83 84# Measure the stats until getting 10 samples or exceeding 30 seconds. addStats 85# is called once per second for now. 86NUM_DECODE_TIME_SAMPLES = 10 87TIMEOUT = 60 88 89 90class video_WebRtcPerf(test.test): 91 """ 92 The test outputs the decode time, cpu usage and the power consumption for 93 WebRTC to performance dashboard. 94 """ 95 version = 1 96 arc_mode = None 97 98 99 def initialize(self): 100 self._service_stopper = None 101 self._original_governors = None 102 self._backlight = None 103 104 105 def cleanup(self): 106 if self._backlight: 107 self._backlight.restore() 108 if self._service_stopper: 109 self._service_stopper.restore_services() 110 if self._original_governors: 111 utils.restore_scaling_governor_states(self._original_governors) 112 113 super(video_WebRtcPerf, self).cleanup() 114 115 116 def start_loopback(self, cr): 117 """ 118 Opens WebRTC loopback page. 119 120 @param cr: Autotest Chrome instance. 121 """ 122 cr.browser.platform.SetHTTPServerDirectories(self.bindir) 123 124 tab = cr.browser.tabs[0] 125 tab.Navigate(cr.browser.platform.http_server.UrlOf( 126 os.path.join(self.bindir, 'loopback.html'))) 127 tab.WaitForDocumentReadyStateToBeComplete() 128 129 130 def open_stats_page(self, cr): 131 """ 132 Opens WebRTC internal statistics page. 133 134 @param cr: Autotest Chrome instance. 135 136 @returns the tab of the stats page. 137 """ 138 tab = cr.browser.tabs.New() 139 tab.Navigate(WEBRTC_INTERNALS_URL) 140 tab.WaitForDocumentReadyStateToBeComplete() 141 # Inject stats callback function. 142 tab.EvaluateJavaScript(ADD_STATS_JAVASCRIPT) 143 return tab 144 145 146 @helper_logger.video_log_wrapper 147 def run_once(self, decode_time_test=False, cpu_test=False, 148 power_test=False, arc_mode=None): 149 """ 150 Runs the video_WebRtcPerf test. 151 152 @param decode_time_test: Pass True to run decode time test. 153 @param cpu_test: Pass True to run CPU usage test. 154 @param power_test: Pass True to run power consumption test. 155 @param arc_mode: if 'enabled', run the test with Android enabled. 156 """ 157 # Download test video. 158 url = DOWNLOAD_BASE + VIDEO_NAME 159 local_path = os.path.join(self.bindir, VIDEO_NAME) 160 file_utils.download_file(url, local_path) 161 self.arc_mode = arc_mode 162 163 if decode_time_test: 164 keyvals = self.test_decode_time(local_path) 165 # The first value is max decode time. The second value is median 166 # decode time. Construct a dictionary containing one value to log 167 # the result. 168 max_decode_time = { 169 key:value[0] for (key, value) in keyvals.items()} 170 self.log_result(max_decode_time, MAX_DECODE_TIME_DESCRIPTION, 171 'milliseconds') 172 median_decode_time = { 173 key:value[1] for (key, value) in keyvals.items()} 174 self.log_result(median_decode_time, MEDIAN_DECODE_TIME_DESCRIPTION, 175 'milliseconds') 176 177 if cpu_test: 178 keyvals = self.test_cpu_usage(local_path) 179 self.log_result(keyvals, CPU_USAGE_DESCRIPTION, 'percent') 180 181 if power_test: 182 keyvals = self.test_power(local_path) 183 self.log_result(keyvals, POWER_DESCRIPTION , 'W') 184 185 186 def test_webrtc(self, local_path, gather_result): 187 """ 188 Runs the webrtc test with and without hardware acceleration. 189 190 @param local_path: the path to the video file. 191 @param gather_result: a function to run and return the test result 192 after chrome opens. The input parameter of the funciton is 193 Autotest chrome instance. 194 195 @return a dictionary that contains test the result. 196 """ 197 keyvals = {} 198 EXTRA_BROWSER_ARGS.append(FAKE_FILE_ARG % local_path) 199 200 with chrome.Chrome(extra_browser_args=EXTRA_BROWSER_ARGS +\ 201 [helper_logger.chrome_vmodule_flag()], 202 arc_mode=self.arc_mode, 203 init_network_controller=True) as cr: 204 # On daisy, Chrome freezes about 30 seconds after login because of 205 # TPM error. See http://crbug.com/588579. 206 if utils.get_board() == 'daisy': 207 logging.warning('Delay 30s for issue 588579 on daisy') 208 time.sleep(30) 209 # Open WebRTC loopback page and start the loopback. 210 self.start_loopback(cr) 211 result = gather_result(cr) 212 213 # Check if decode is hardware accelerated. 214 if histogram_verifier.is_bucket_present( 215 cr, 216 constants.RTC_INIT_HISTOGRAM, 217 constants.RTC_VIDEO_INIT_BUCKET): 218 keyvals[WEBRTC_WITH_HW_ACCELERATION] = result 219 else: 220 logging.info("Can not use hardware decoding.") 221 keyvals[WEBRTC_WITHOUT_HW_ACCELERATION] = result 222 return keyvals 223 224 # Start chrome with disabled video hardware decode flag. 225 with chrome.Chrome(extra_browser_args= 226 DISABLE_ACCELERATED_VIDEO_DECODE_BROWSER_ARGS + 227 EXTRA_BROWSER_ARGS, arc_mode=self.arc_mode, 228 init_network_controller=True) as cr: 229 230 # crbug/753292 - enforce the idle checks after login 231 if not utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, 232 CPU_IDLE_USAGE): 233 logging.warning('Could not get idle CPU post login.') 234 if not utils.wait_for_cool_machine(): 235 logging.warning('Could not get cold machine post login.') 236 237 if utils.get_board() == 'daisy': 238 logging.warning('Delay 30s for issue 588579 on daisy') 239 time.sleep(30) 240 # Open the webrtc loopback page and start the loopback. 241 self.start_loopback(cr) 242 result = gather_result(cr) 243 244 # Make sure decode is not hardware accelerated. 245 if histogram_verifier.is_bucket_present( 246 cr, 247 constants.RTC_INIT_HISTOGRAM, 248 constants.RTC_VIDEO_INIT_BUCKET): 249 raise error.TestError('HW decode should not be used.') 250 keyvals[WEBRTC_WITHOUT_HW_ACCELERATION] = result 251 252 return keyvals 253 254 255 def test_cpu_usage(self, local_path): 256 """ 257 Runs the video cpu usage test. 258 259 @param local_path: the path to the video file. 260 261 @return a dictionary that contains the test result. 262 """ 263 def get_cpu_usage(cr): 264 time.sleep(STABILIZATION_DURATION) 265 cpu_usage_start = utils.get_cpu_usage() 266 time.sleep(MEASUREMENT_DURATION) 267 cpu_usage_end = utils.get_cpu_usage() 268 return utils.compute_active_cpu_time(cpu_usage_start, 269 cpu_usage_end) * 100 270 271 # crbug/753292 - APNG login pictures increase CPU usage. Move the more 272 # strict idle checks after the login phase. 273 utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, CPU_IDLE_USAGE) 274 utils.wait_for_cool_machine() 275 if not utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, 276 CPU_IDLE_USAGE): 277 logging.warning('Could not get idle CPU pre login.') 278 if not utils.wait_for_cool_machine(): 279 logging.warning('Could not get cold machine pre login.') 280 281 # Stop the thermal service that may change the cpu frequency. 282 self._service_stopper = service_stopper.ServiceStopper(THERMAL_SERVICES) 283 self._service_stopper.stop_services() 284 # Set the scaling governor to performance mode to set the cpu to the 285 # highest frequency available. 286 self._original_governors = utils.set_high_performance_mode() 287 return self.test_webrtc(local_path, get_cpu_usage) 288 289 290 def test_power(self, local_path): 291 """ 292 Runs the video power consumption test. 293 294 @param local_path: the path to the video file. 295 296 @return a dictionary that contains the test result. 297 """ 298 self._backlight = power_utils.Backlight() 299 self._backlight.set_default() 300 self._service_stopper = service_stopper.ServiceStopper( 301 service_stopper.ServiceStopper.POWER_DRAW_SERVICES) 302 self._service_stopper.stop_services() 303 304 current_power_status = power_status.get_status() 305 # We expect the DUT is powered by battery now. But this is not always 306 # true due to other bugs. Disable this test temporarily as workaround. 307 # TODO(kcwu): remove this workaround after AC control is stable 308 # crbug.com/723968 309 if current_power_status.on_ac(): 310 logging.warning('Still powered by AC. Skip this test') 311 return {} 312 # Verify that the battery is sufficiently charged. 313 current_power_status.assert_battery_state(BATTERY_INITIAL_CHARGED_MIN) 314 315 measurements = [power_status.SystemPower( 316 current_power_status.battery_path)] 317 318 def get_power(cr): 319 power_logger = power_status.PowerLogger(measurements) 320 power_logger.start() 321 time.sleep(STABILIZATION_DURATION) 322 start_time = time.time() 323 time.sleep(MEASUREMENT_DURATION) 324 power_logger.checkpoint('result', start_time) 325 keyval = power_logger.calc() 326 return keyval['result_' + measurements[0].domain + '_pwr'] 327 328 return self.test_webrtc(local_path, get_power) 329 330 331 def test_decode_time(self, local_path): 332 """ 333 Runs the decode time test. 334 335 @param local_path: the path to the video file. 336 337 @return a dictionary that contains the test result. 338 """ 339 def get_decode_time(cr): 340 tab = self.open_stats_page(cr) 341 # Collect the decode time until there are enough samples. 342 start_time = time.time() 343 max_decode_time_list = [] 344 decode_time_list = [] 345 while (time.time() - start_time < TIMEOUT and 346 len(decode_time_list) < NUM_DECODE_TIME_SAMPLES): 347 time.sleep(1) 348 max_decode_time_list = [] 349 decode_time_list = [] 350 try: 351 max_decode_time_list = [int(x) for x in 352 tab.EvaluateJavaScript(GOOG_MAX_DECODE_MS)] 353 decode_time_list = [int(x) for x in 354 tab.EvaluateJavaScript(GOOG_DECODE_MS)] 355 except: 356 pass 357 # Output the values if they are valid. 358 if len(max_decode_time_list) < NUM_DECODE_TIME_SAMPLES: 359 raise error.TestError('Not enough ' + GOOG_MAX_DECODE_MS) 360 if len(decode_time_list) < NUM_DECODE_TIME_SAMPLES: 361 raise error.TestError('Not enough ' + GOOG_DECODE_MS) 362 max_decode_time = max(max_decode_time_list) 363 decode_time_median = self.get_median(decode_time_list) 364 logging.info("Max decode time list=%s", str(max_decode_time_list)) 365 logging.info("Decode time list=%s", str(decode_time_list)) 366 logging.info("Maximum decode time=%d, median=%d", max_decode_time, 367 decode_time_median) 368 return (max_decode_time, decode_time_median) 369 370 return self.test_webrtc(local_path, get_decode_time) 371 372 373 def get_median(self, seq): 374 """ 375 Calculates the median of a sequence of numbers. 376 377 @param seq: a list with numbers. 378 379 @returns the median of the numbers. 380 """ 381 seq.sort() 382 size = len(seq) 383 if size % 2 != 0: 384 return seq[size / 2] 385 return (seq[size / 2] + seq[size / 2 - 1]) / 2.0 386 387 def log_result(self, keyvals, description, units): 388 """ 389 Logs the test result output to the performance dashboard. 390 391 @param keyvals: a dictionary that contains results returned by 392 test_webrtc. 393 @param description: a string that describes the video and test result 394 and it will be part of the entry name in the dashboard. 395 @param units: the units of test result. 396 """ 397 result_with_hw = keyvals.get(WEBRTC_WITH_HW_ACCELERATION) 398 if result_with_hw: 399 self.output_perf_value( 400 description= 'hw_' + description, value=result_with_hw, 401 units=units, higher_is_better=False) 402 403 result_without_hw = keyvals.get(WEBRTC_WITHOUT_HW_ACCELERATION) 404 if result_without_hw: 405 self.output_perf_value( 406 description= 'sw_' + description, value=result_without_hw, 407 units=units, higher_is_better=False) 408