1# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import time 8 9from autotest_lib.client.bin import test, utils 10from autotest_lib.client.common_lib import error 11from autotest_lib.client.common_lib import file_utils 12from autotest_lib.client.common_lib.cros import chrome 13from autotest_lib.client.cros import service_stopper 14from autotest_lib.client.cros.power import power_status, power_utils 15from autotest_lib.client.cros.video import histogram_verifier 16from autotest_lib.client.cros.video import constants 17 18 19EXTRA_BROWSER_ARGS = ['--use-fake-device-for-media-stream', 20 '--use-fake-ui-for-media-stream'] 21FAKE_FILE_ARG = '--use-file-for-fake-video-capture="%s"' 22DISABLE_ACCELERATED_VIDEO_DECODE_BROWSER_ARGS = [ 23 '--disable-accelerated-video-decode'] 24 25DOWNLOAD_BASE = ('http://commondatastorage.googleapis.com/' 26 'chromiumos-test-assets-public/crowd/') 27VIDEO_NAME = 'crowd720_25frames.y4m' 28 29WEBRTC_INTERNALS_URL = 'chrome://webrtc-internals' 30 31WEBRTC_WITH_HW_ACCELERATION = 'webrtc_with_hw_acceleration' 32WEBRTC_WITHOUT_HW_ACCELERATION = 'webrtc_without_hw_acceleration' 33 34# Measurement duration in seconds. 35MEASUREMENT_DURATION = 30 36# Time to exclude from calculation after start the loopback [seconds]. 37STABILIZATION_DURATION = 10 38 39# Time in seconds to wait for cpu idle until giving up. 40WAIT_FOR_IDLE_CPU_TIMEOUT = 60.0 41# Maximum percent of cpu usage considered as idle. 42CPU_IDLE_USAGE = 0.1 43 44MAX_DECODE_TIME_DESCRIPTION = 'decode_time.max' 45MEDIAN_DECODE_TIME_DESCRIPTION = 'decode_time.percentile_0.50' 46CPU_USAGE_DESCRIPTION = 'video_cpu_usage' 47POWER_DESCRIPTION = 'video_mean_energy_rate' 48 49# Minimum battery charge percentage to run the power test. 50BATTERY_INITIAL_CHARGED_MIN = 10 51 52# These are the variable names in WebRTC internals. 53# Maximum decode time of the frames of the last 10 seconds. 54GOOG_MAX_DECODE_MS = 'googMaxDecodeMs' 55# The decode time of the last frame. 56GOOG_DECODE_MS = 'googDecodeMs' 57 58# Javascript function to get the decode time. addLegacyStats is a function 59# called by Chrome to pass WebRTC statistics every second. 60ADD_STATS_JAVASCRIPT = ( 61 'var googMaxDecodeMs = new Array();' 62 'var googDecodeMs = new Array();' 63 'addLegacyStats = function(data) {' 64 ' reports = data.reports;' 65 ' for (var i=0; i < reports.length; i++) {' 66 ' if (reports[i].type == "ssrc") {' 67 ' values = reports[i].stats.values;' 68 ' for (var j=0; j < values.length; j++) {' 69 ' if (values[j] == "googMaxDecodeMs")' 70 ' googMaxDecodeMs[googMaxDecodeMs.length] = values[j+1];' 71 ' else if (values[j] == "googDecodeMs")' 72 ' googDecodeMs[googDecodeMs.length] = values[j+1];' 73 ' }' 74 ' }' 75 ' }' 76 '}') 77 78# Measure the stats until getting 10 samples or exceeding 30 seconds. 79# addLegacyStats is called once per second for now. 80NUM_DECODE_TIME_SAMPLES = 10 81TIMEOUT = 60 82 83 84class video_WebRtcPerf(test.test): 85 """ 86 The test outputs the decode time, cpu usage and the power consumption for 87 WebRTC to performance dashboard. 88 """ 89 version = 1 90 arc_mode = None 91 92 93 def initialize(self): 94 """Initializes the test.""" 95 self._service_stopper = None 96 self._original_governors = None 97 self._backlight = None 98 99 100 def cleanup(self): 101 """Restores device setting for performance test.""" 102 if self._backlight: 103 self._backlight.restore() 104 if self._service_stopper: 105 self._service_stopper.restore_services() 106 if self._original_governors: 107 utils.restore_scaling_governor_states(self._original_governors) 108 109 super(video_WebRtcPerf, self).cleanup() 110 111 112 def start_loopback(self, cr): 113 """ 114 Opens WebRTC loopback page. 115 116 @param cr: Autotest Chrome instance. 117 """ 118 cr.browser.platform.SetHTTPServerDirectories(self.bindir) 119 120 tab = cr.browser.tabs[0] 121 tab.Navigate(cr.browser.platform.http_server.UrlOf( 122 os.path.join(self.bindir, 'loopback.html'))) 123 tab.WaitForDocumentReadyStateToBeComplete() 124 125 126 def open_stats_page(self, cr): 127 """ 128 Opens WebRTC internal statistics page. 129 130 @param cr: Autotest Chrome instance. 131 132 @returns the tab of the stats page. 133 """ 134 tab = cr.browser.tabs.New() 135 tab.Navigate(WEBRTC_INTERNALS_URL) 136 tab.WaitForDocumentReadyStateToBeComplete() 137 # Switch to legacy mode to get googDecodeMs and googMaxDecodeMs, refer 138 # crbug.com/803014. 139 tab.EvaluateJavaScript("currentGetStatsMethod = OPTION_GETSTATS_LEGACY") 140 # Inject stats callback function. 141 tab.EvaluateJavaScript(ADD_STATS_JAVASCRIPT) 142 return tab 143 144 145 def run_once(self, decode_time_test=False, cpu_test=False, 146 power_test=False, arc_mode=None): 147 """ 148 Runs the video_WebRtcPerf test. 149 150 @param decode_time_test: Pass True to run decode time test. 151 @param cpu_test: Pass True to run CPU usage test. 152 @param power_test: Pass True to run power consumption test. 153 @param arc_mode: if 'enabled', run the test with Android enabled. 154 """ 155 # Download test video. 156 url = DOWNLOAD_BASE + VIDEO_NAME 157 local_path = os.path.join(self.bindir, VIDEO_NAME) 158 file_utils.download_file(url, local_path) 159 self.arc_mode = arc_mode 160 161 if decode_time_test: 162 keyvals = self.test_decode_time(local_path) 163 # The first value is max decode time. The second value is median 164 # decode time. Construct a dictionary containing one value to log 165 # the result. 166 max_decode_time = { 167 key:value[0] for (key, value) in keyvals.items()} 168 self.log_result(max_decode_time, MAX_DECODE_TIME_DESCRIPTION, 169 'milliseconds') 170 median_decode_time = { 171 key:value[1] for (key, value) in keyvals.items()} 172 self.log_result(median_decode_time, MEDIAN_DECODE_TIME_DESCRIPTION, 173 'milliseconds') 174 175 if cpu_test: 176 keyvals = self.test_cpu_usage(local_path) 177 self.log_result(keyvals, CPU_USAGE_DESCRIPTION, 'percent') 178 179 if power_test: 180 keyvals = self.test_power(local_path) 181 self.log_result(keyvals, POWER_DESCRIPTION , 'W') 182 183 184 def test_webrtc(self, local_path, gather_result): 185 """ 186 Runs the webrtc test with and without hardware acceleration. 187 188 @param local_path: the path to the video file. 189 @param gather_result: a function to run and return the test result 190 after chrome opens. The input parameter of the funciton is 191 Autotest chrome instance. 192 193 @return a dictionary that contains test the result. 194 """ 195 keyvals = {} 196 EXTRA_BROWSER_ARGS.append(FAKE_FILE_ARG % local_path) 197 198 with chrome.Chrome(extra_browser_args=EXTRA_BROWSER_ARGS, 199 arc_mode=self.arc_mode, 200 init_network_controller=True) as cr: 201 # On daisy, Chrome freezes about 30 seconds after login because of 202 # TPM error. See http://crbug.com/588579. 203 if utils.get_board() == 'daisy': 204 logging.warning('Delay 30s for issue 588579 on daisy') 205 time.sleep(30) 206 # Open WebRTC loopback page and start the loopback. 207 self.start_loopback(cr) 208 result = gather_result(cr) 209 210 # Check if decode is hardware accelerated. 211 if histogram_verifier.is_bucket_present( 212 cr, 213 constants.RTC_INIT_HISTOGRAM, 214 constants.RTC_VIDEO_INIT_BUCKET): 215 keyvals[WEBRTC_WITH_HW_ACCELERATION] = result 216 else: 217 logging.info("Can not use hardware decoding.") 218 keyvals[WEBRTC_WITHOUT_HW_ACCELERATION] = result 219 return keyvals 220 221 # Start chrome with disabled video hardware decode flag. 222 with chrome.Chrome(extra_browser_args= 223 DISABLE_ACCELERATED_VIDEO_DECODE_BROWSER_ARGS + 224 EXTRA_BROWSER_ARGS, arc_mode=self.arc_mode, 225 init_network_controller=True) as cr: 226 227 # crbug/753292 - enforce the idle checks after login 228 if not utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, 229 CPU_IDLE_USAGE): 230 logging.warning('Could not get idle CPU post login.') 231 if not utils.wait_for_cool_machine(): 232 logging.warning('Could not get cold machine post login.') 233 234 if utils.get_board() == 'daisy': 235 logging.warning('Delay 30s for issue 588579 on daisy') 236 time.sleep(30) 237 # Open the webrtc loopback page and start the loopback. 238 self.start_loopback(cr) 239 result = gather_result(cr) 240 241 # Make sure decode is not hardware accelerated. 242 if histogram_verifier.is_bucket_present( 243 cr, 244 constants.RTC_INIT_HISTOGRAM, 245 constants.RTC_VIDEO_INIT_BUCKET): 246 raise error.TestError('HW decode should not be used.') 247 keyvals[WEBRTC_WITHOUT_HW_ACCELERATION] = result 248 249 return keyvals 250 251 252 def test_cpu_usage(self, local_path): 253 """ 254 Runs the video cpu usage test. 255 256 @param local_path: the path to the video file. 257 258 @return a dictionary that contains the test result. 259 """ 260 def get_cpu_usage(cr): 261 time.sleep(STABILIZATION_DURATION) 262 cpu_usage_start = utils.get_cpu_usage() 263 time.sleep(MEASUREMENT_DURATION) 264 cpu_usage_end = utils.get_cpu_usage() 265 return utils.compute_active_cpu_time(cpu_usage_start, 266 cpu_usage_end) * 100 267 268 # crbug/753292 - APNG login pictures increase CPU usage. Move the more 269 # strict idle checks after the login phase. 270 utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, CPU_IDLE_USAGE) 271 utils.wait_for_cool_machine() 272 if not utils.wait_for_idle_cpu(WAIT_FOR_IDLE_CPU_TIMEOUT, 273 CPU_IDLE_USAGE): 274 logging.warning('Could not get idle CPU pre login.') 275 if not utils.wait_for_cool_machine(): 276 logging.warning('Could not get cold machine pre login.') 277 278 # Stop the thermal service that may change the cpu frequency. 279 self._service_stopper = service_stopper.get_thermal_service_stopper() 280 self._service_stopper.stop_services() 281 # Set the scaling governor to performance mode to set the cpu to the 282 # highest frequency available. 283 self._original_governors = utils.set_high_performance_mode() 284 return self.test_webrtc(local_path, get_cpu_usage) 285 286 287 def test_power(self, local_path): 288 """ 289 Runs the video power consumption test. 290 291 @param local_path: the path to the video file. 292 293 @return a dictionary that contains the test result. 294 """ 295 self._backlight = power_utils.Backlight() 296 self._backlight.set_default() 297 self._service_stopper = service_stopper.ServiceStopper( 298 service_stopper.ServiceStopper.POWER_DRAW_SERVICES) 299 self._service_stopper.stop_services() 300 301 current_power_status = power_status.get_status() 302 # We expect the DUT is powered by battery now. But this is not always 303 # true due to other bugs. Disable this test temporarily as workaround. 304 # TODO(kcwu): remove this workaround after AC control is stable 305 # crbug.com/723968 306 if current_power_status.on_ac(): 307 logging.warning('Still powered by AC. Skip this test') 308 return {} 309 # Verify that the battery is sufficiently charged. 310 current_power_status.assert_battery_state(BATTERY_INITIAL_CHARGED_MIN) 311 312 measurements = [power_status.SystemPower( 313 current_power_status.battery_path)] 314 315 def get_power(cr): 316 power_logger = power_status.PowerLogger(measurements) 317 power_logger.start() 318 time.sleep(STABILIZATION_DURATION) 319 start_time = time.time() 320 time.sleep(MEASUREMENT_DURATION) 321 power_logger.checkpoint('result', start_time) 322 keyval = power_logger.calc() 323 # save_results() will save <fname_prefix>_raw.txt and 324 # <fname_prefix>_summary.txt, where the former contains raw data. 325 fname_prefix = 'result_%.0f' % time.time() 326 power_logger.save_results(self.resultsdir, fname_prefix) 327 metric_name = 'result_' + measurements[0].domain 328 with open(os.path.join( 329 self.resultsdir, fname_prefix + '_raw.txt')) as f: 330 for line in f.readlines(): 331 if line.startswith(metric_name): 332 split_data = line.split('\t') 333 # split_data[0] is metric_name, [1:] are raw data. 334 return [float(data) for data in split_data[1:]] 335 # Return a list contains the average power only for fallback. 336 return [keyval[metric_name + '_pwr_avg']] 337 338 return self.test_webrtc(local_path, get_power) 339 340 341 def test_decode_time(self, local_path): 342 """ 343 Runs the decode time test. 344 345 @param local_path: the path to the video file. 346 347 @return a dictionary that contains the test result. 348 """ 349 def get_decode_time(cr): 350 tab = self.open_stats_page(cr) 351 # Collect the decode time until there are enough samples. 352 start_time = time.time() 353 max_decode_time_list = [] 354 decode_time_list = [] 355 while (time.time() - start_time < TIMEOUT and 356 len(decode_time_list) < NUM_DECODE_TIME_SAMPLES): 357 time.sleep(1) 358 max_decode_time_list = [] 359 decode_time_list = [] 360 try: 361 max_decode_time_list = [int(x) for x in 362 tab.EvaluateJavaScript(GOOG_MAX_DECODE_MS)] 363 decode_time_list = [int(x) for x in 364 tab.EvaluateJavaScript(GOOG_DECODE_MS)] 365 except: 366 pass 367 # Output the values if they are valid. 368 if len(max_decode_time_list) < NUM_DECODE_TIME_SAMPLES: 369 raise error.TestError('Not enough ' + GOOG_MAX_DECODE_MS) 370 if len(decode_time_list) < NUM_DECODE_TIME_SAMPLES: 371 raise error.TestError('Not enough ' + GOOG_DECODE_MS) 372 max_decode_time = max(max_decode_time_list) 373 decode_time_median = self.get_median(decode_time_list) 374 logging.info("Max decode time list=%s", str(max_decode_time_list)) 375 logging.info("Decode time list=%s", str(decode_time_list)) 376 logging.info("Maximum decode time=%d, median=%d", max_decode_time, 377 decode_time_median) 378 return (max_decode_time, decode_time_median) 379 380 return self.test_webrtc(local_path, get_decode_time) 381 382 383 def get_median(self, seq): 384 """ 385 Calculates the median of a sequence of numbers. 386 387 @param seq: a list with numbers. 388 389 @returns the median of the numbers. 390 """ 391 seq.sort() 392 size = len(seq) 393 if size % 2 != 0: 394 return seq[size / 2] 395 return (seq[size / 2] + seq[size / 2 - 1]) / 2.0 396 397 def log_result(self, keyvals, description, units): 398 """ 399 Logs the test result output to the performance dashboard. 400 401 @param keyvals: a dictionary that contains results returned by 402 test_webrtc. 403 @param description: a string that describes the video and test result 404 and it will be part of the entry name in the dashboard. 405 @param units: the units of test result. 406 """ 407 result_with_hw = keyvals.get(WEBRTC_WITH_HW_ACCELERATION) 408 if result_with_hw: 409 self.output_perf_value( 410 description= 'hw_' + description, value=result_with_hw, 411 units=units, higher_is_better=False) 412 413 result_without_hw = keyvals.get(WEBRTC_WITHOUT_HW_ACCELERATION) 414 if result_without_hw: 415 self.output_perf_value( 416 description= 'sw_' + description, value=result_without_hw, 417 units=units, higher_is_better=False) 418