1# Copyright (c) 2017 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import csv, datetime, glob, json, math, os, re, time 6 7from autotest_lib.client.bin import utils 8from autotest_lib.client.common_lib import error 9from autotest_lib.client.common_lib.cros import tpm_utils 10from autotest_lib.server import test 11from autotest_lib.server.cros import cfm_jmidata_log_collector 12from autotest_lib.server.cros.multimedia import remote_facade_factory 13 14_SHORT_TIMEOUT = 5 15_MEASUREMENT_DURATION_SECONDS = 10 16_TOTAL_TEST_DURATION_SECONDS = 900 17_PERF_RESULT_FILE = 'perf.csv' 18_JMI_RESULT_FILE = 'jmidata.json' 19 20_BASE_DIR = '/home/chronos/user/Storage/ext/' 21_EXT_ID = 'ikfcpmgefdpheiiomgmhlmmkihchmdlj' 22_JMI_DIR = '/0*/File\ System/000/t/00/*' 23_JMI_SOURCE_DIR = _BASE_DIR + _EXT_ID + _JMI_DIR 24 25 26class enterprise_CFM_Perf(test.test): 27 """This is a server test which clears device TPM and runs 28 enterprise_RemoraRequisition client test to enroll the device in to hotrod 29 mode. After enrollment is successful, it collects and logs cpu, memory and 30 temperature data from the device under test.""" 31 version = 1 32 33 34 def _cpu_usage(self): 35 """Returns cpu usage in %.""" 36 cpu_usage_start = self.system_facade.get_cpu_usage() 37 time.sleep(_MEASUREMENT_DURATION_SECONDS) 38 cpu_usage_end = self.system_facade.get_cpu_usage() 39 return self.system_facade.compute_active_cpu_time(cpu_usage_start, 40 cpu_usage_end) * 100 41 42 43 def _memory_usage(self): 44 """Returns total used memory in %.""" 45 total_memory = self.system_facade.get_mem_total() 46 return ((total_memory - self.system_facade.get_mem_free()) 47 * 100 / total_memory) 48 49 50 def _temperature_data(self): 51 """Returns temperature sensor data in fahrenheit.""" 52 ectool = self.client.run('ectool version', ignore_status=True) 53 if not ectool.exit_status: 54 ec_temp = self.system_facade.get_ec_temperatures() 55 return ec_temp[1] 56 else: 57 temp_sensor_name = 'temp0' 58 if not temp_sensor_name: 59 return 0 60 MOSYS_OUTPUT_RE = re.compile('(\w+)="(.*?)"') 61 values = {} 62 cmd = 'mosys -k sensor print thermal %s' % temp_sensor_name 63 for kv in MOSYS_OUTPUT_RE.finditer(self.client.run_output(cmd)): 64 key, value = kv.groups() 65 if key == 'reading': 66 value = int(value) 67 values[key] = value 68 return values['reading'] 69 70 71 def enroll_device_and_start_hangout(self): 72 """Enroll device into CFM and start hangout session.""" 73 current_date = datetime.datetime.now().strftime("%Y-%m-%d") 74 hangout_name = current_date + '-cfm-perf' 75 76 self.cfm_facade.enroll_device() 77 self.cfm_facade.skip_oobe_after_enrollment() 78 self.cfm_facade.start_new_hangout_session(hangout_name) 79 80 81 def enroll_device_and_join_meeting(self): 82 """Enroll device into CFM and join a meeting session.""" 83 self.cfm_facade.enroll_device() 84 self.cfm_facade.skip_oobe_after_enrollment() 85 86 self.cfm_facade.wait_for_meetings_landing_page() 87 # Daily meeting for perf testing with 9 remote participants. 88 meeting_code = 'nis-rhmz-dyh' 89 self.cfm_facade.join_meeting_session(meeting_code) 90 91 92 def collect_perf_data(self): 93 """Use system facade to collect performance data from the DUT using 94 xmlrpc and save it to csv file in results directory. Data collected 95 includes: 96 1. CPU usage 97 2. Memory usage 98 3. Thermal temperature 99 4. Timestamp 100 5. Board name 101 6. Build id 102 """ 103 start_time = time.time() 104 perf_keyval = {} 105 cpu_usage_list = list() 106 memory_usage_list = list() 107 temperature_list = list() 108 board_name = self.system_facade.get_current_board() 109 build_id = self.system_facade.get_chromeos_release_version() 110 perf_file = open(os.path.join(self.resultsdir, _PERF_RESULT_FILE), 'w') 111 writer = csv.writer(perf_file) 112 writer.writerow(['cpu', 'memory', 'temperature', 'timestamp', 'board', 113 'build']) 114 while (time.time() - start_time) < _TOTAL_TEST_DURATION_SECONDS: 115 perf_keyval['cpu_usage'] = self._cpu_usage() 116 perf_keyval['memory_usage'] = self._memory_usage() 117 perf_keyval['temperature'] = self._temperature_data() 118 writer.writerow([perf_keyval['cpu_usage'], 119 perf_keyval['memory_usage'], 120 perf_keyval['temperature'], 121 time.strftime('%Y/%m/%d %H:%M:%S'), 122 board_name, 123 build_id]) 124 self.write_perf_keyval(perf_keyval) 125 cpu_usage_list.append(perf_keyval['cpu_usage']) 126 memory_usage_list.append(perf_keyval['memory_usage']) 127 temperature_list.append(perf_keyval['temperature']) 128 time.sleep(_MEASUREMENT_DURATION_SECONDS) 129 perf_file.close() 130 utils.write_keyval(os.path.join(self.resultsdir, os.pardir), 131 {'perf_csv_folder': self.resultsdir}) 132 self.upload_perf_data(cpu_usage_list, 133 memory_usage_list, 134 temperature_list) 135 136 137 def upload_perf_data(self, cpu_usage, memory_usage, temperature): 138 """Write perf results to results-chart.json file for Perf Dashboard. 139 140 @param cpu_usage: list of cpu usage values 141 @param memory_usage: list of memory usage values 142 @param temperature: list of temperature values 143 """ 144 avg_cpu_usage = sum(cpu_usage)/len(cpu_usage) 145 avg_memory_usage = sum(memory_usage)/len(memory_usage) 146 avg_temp = sum(temperature)/len(temperature) 147 148 peak_cpu_usage = max(cpu_usage) 149 peak_memory_usage = max(memory_usage) 150 peak_temp = max(temperature) 151 152 self.output_perf_value(description='average_cpu_usage', 153 value=avg_cpu_usage, units='percent', higher_is_better=False) 154 self.output_perf_value(description='average_memory_usage', 155 value=avg_memory_usage, units='percent', higher_is_better=False) 156 self.output_perf_value(description='average_temperature', 157 value=avg_temp, units='Celsius', higher_is_better=False) 158 159 self.output_perf_value(description='cpu_usage', 160 value=cpu_usage, units='percent', higher_is_better=False) 161 self.output_perf_value(description='memory_usage', 162 value=memory_usage, units='percent', higher_is_better=False) 163 self.output_perf_value(description='temperature', 164 value=temperature, units='Celsius', higher_is_better=False) 165 166 self.output_perf_value(description='peak_cpu_usage', 167 value=peak_cpu_usage, units='percent', higher_is_better=False) 168 self.output_perf_value(description='peak_memory_usage', 169 value=peak_memory_usage, units='percent', 170 higher_is_better=False) 171 self.output_perf_value(description='peak_temperature', 172 value=peak_temp, units='Celsius', higher_is_better=False) 173 174 175 def _get_average(self, data_type, jmidata): 176 """Computes mean of a list of numbers. 177 178 @param data_type: Type of data to be retrieved from jmi data log. 179 @param jmidata: Raw jmi data log to parse. 180 @return Mean computed from the list of numbers. 181 """ 182 data = self._get_data_from_jmifile(data_type, jmidata) 183 if not data: 184 return 0 185 return float(sum(data)) / len(data) 186 187 188 def _get_std_dev(self, data_type, jmidata): 189 """Computes standard deviation of a list of numbers. 190 191 @param data_type: Type of data to be retrieved from jmi data log. 192 @param jmidata: Raw jmi data log to parse. 193 @return Standard deviation computed from the list of numbers. 194 """ 195 data = self._get_data_from_jmifile(data_type, jmidata) 196 n = len(data) 197 if not data or n == 1: 198 return 0 199 mean = float(sum(data)) / n 200 variance = sum([(elem - mean) ** 2 for elem in data]) / (n -1) 201 return math.sqrt(variance) 202 203 204 def _get_max_value(self, data_type, jmidata): 205 """Computes maximum value of a list of numbers. 206 207 @param data_type: Type of data to be retrieved from jmi data log. 208 @param jmidata: Raw jmi data log to parse. 209 @return Maxium value from the list of numbers. 210 """ 211 data = self._get_data_from_jmifile(data_type, jmidata) 212 if not data: 213 return 0 214 return max(data) 215 216 217 def _get_sum(self, data_type, jmidata): 218 """Computes sum of a list of numbers. 219 220 @param data_type: Type of data to be retrieved from jmi data log. 221 @param jmidata: Raw jmi data log to parse. 222 @return Sum computed from the list of numbers. 223 """ 224 data = self._get_data_from_jmifile(data_type, jmidata) 225 if not data: 226 return 0 227 return sum(data) 228 229 230 def _get_last_value(self, data_type, jmidata): 231 """Gets last value of a list of numbers. 232 233 @param data_type: Type of data to be retrieved from jmi data log. 234 @param jmidata: Raw jmi data log to parse. 235 @return Mean computed from the list of numbers. 236 """ 237 data = self._get_data_from_jmifile(data_type, jmidata) 238 if not data: 239 return 0 240 return data[-1] 241 242 243 def _get_data_from_jmifile(self, data_type, jmidata): 244 """Gets data from jmidata log for given data type. 245 246 @param data_type: Type of data to be retrieved from jmi data log. 247 @param jmidata: Raw jmi data log to parse. 248 @return Data for given data type from jmidata log. 249 """ 250 return cfm_jmidata_log_collector.GetDataFromLogs( 251 self, data_type, jmidata) 252 253 254 def _get_file_to_parse(self): 255 """Copy jmi logs from client to test's results directory. 256 257 @return The newest jmi log file. 258 """ 259 self.client.get_file(_JMI_SOURCE_DIR, self.resultsdir) 260 source_jmi_files = self.resultsdir + '/0*' 261 if not source_jmi_files: 262 raise error.TestNAError('JMI data file not found.') 263 newest_file = max(glob.iglob(source_jmi_files), key=os.path.getctime) 264 return newest_file 265 266 267 def _dump_raw_jmi_data(self, jmidata): 268 """ 269 Write the raw JMI data into the _JMI_RESULT_FILE for later processing. 270 """ 271 data_types = [ 272 'frames_decoded', 273 'frames_encoded', 274 'adaptation_changes', 275 'average_encode_time', 276 'bandwidth_adaptation', 277 'cpu_adaptation', 278 'video_received_frame_height', 279 'video_sent_frame_height', 280 'framerate_decoded', 281 'framerate_outgoing', 282 'framerate_to_renderer', 283 'framerate_received', 284 'framerate_sent', 285 'video_received_frame_width', 286 'video_sent_frame_width', 287 'video_encode_cpu_usage', 288 'video_packets_sent', 289 'video_packets_lost', 290 'cpu_processors', 291 'cpu_percent', 292 'renderer_cpu_percent', 293 'browser_cpu_percent', 294 'gpu_cpu_percent', 295 'num_active_vid_in_streams', 296 ] 297 298 # Collect all the raw JMI values into a dictionary. 299 results = {} 300 for data_type in data_types: 301 data = self._get_data_from_jmifile(data_type, jmidata) 302 if not data: 303 data = -1 304 results[data_type] = data 305 306 # Dump the dictionary as json into a log file. 307 result_file_path = os.path.join(self.resultsdir, _JMI_RESULT_FILE) 308 with open(result_file_path, 'w') as fp: 309 fp.write(json.dumps(results, indent=2)) 310 311 312 def upload_jmidata(self): 313 """ 314 Write jmidata results to results-chart.json file for Perf Dashboard 315 and also save the raw data. 316 """ 317 jmi_file = self._get_file_to_parse() 318 jmifile_to_parse = open(jmi_file, 'r') 319 jmidata = jmifile_to_parse.read() 320 321 # Start by saving the jmi data separately as raw values in a json file. 322 self._dump_raw_jmi_data(jmidata) 323 324 # Compute and save aggregated stats from JMI. 325 self.output_perf_value(description='sum_vid_in_frames_decoded', 326 value=self._get_sum('frames_decoded', jmidata), units='frames', 327 higher_is_better=True) 328 329 self.output_perf_value(description='sum_vid_out_frames_encoded', 330 value=self._get_sum('frames_encoded', jmidata), units='frames', 331 higher_is_better=True) 332 333 self.output_perf_value(description='vid_out_adapt_changes', 334 value=self._get_last_value('adaptation_changes', jmidata), 335 units='count', higher_is_better=False) 336 337 self.output_perf_value(description='avg_video_out_encode_time', 338 value=self._get_average('average_encode_time', jmidata), 339 units='ms', higher_is_better=False) 340 341 self.output_perf_value(description='video_out_encode_time', 342 value=self._get_data_from_jmifile( 343 'average_encode_time', jmidata), 344 units='ms', higher_is_better=False) 345 346 self.output_perf_value(description='std_dev_video_out_encode_time', 347 value=self._get_std_dev('average_encode_time', jmidata), 348 units='ms', higher_is_better=False) 349 350 self.output_perf_value(description='max_video_out_encode_time', 351 value=self._get_max_value('average_encode_time', jmidata), 352 units='ms', higher_is_better=False) 353 354 self.output_perf_value(description='vid_out_bandwidth_adapt', 355 value=self._get_average('bandwidth_adaptation', jmidata), 356 units='bool', higher_is_better=False) 357 358 self.output_perf_value(description='vid_out_cpu_adapt', 359 value=self._get_average('cpu_adaptation', jmidata), 360 units='bool', higher_is_better=False) 361 362 self.output_perf_value(description='avg_video_in_res', 363 value=self._get_average('video_received_frame_height', jmidata), 364 units='resolution', higher_is_better=True) 365 366 self.output_perf_value(description='video_in_res', 367 value=self._get_data_from_jmifile( 368 'video_received_frame_height', jmidata), 369 units='resolution', higher_is_better=True) 370 371 self.output_perf_value(description='avg_video_out_res', 372 value=self._get_average('video_sent_frame_height', jmidata), 373 units='resolution', higher_is_better=True) 374 375 self.output_perf_value(description='video_out_res', 376 value=self._get_data_from_jmifile( 377 'video_sent_frame_height', jmidata), 378 units='resolution', higher_is_better=True) 379 380 self.output_perf_value(description='std_dev_video_out_res', 381 value=self._get_std_dev('video_sent_frame_height', jmidata), 382 units='resolution', higher_is_better=True) 383 384 self.output_perf_value(description='avg_vid_in_framerate_decoded', 385 value=self._get_average('framerate_decoded', jmidata), 386 units='fps', higher_is_better=True) 387 388 self.output_perf_value(description='vid_in_framerate_decoded', 389 value=self._get_data_from_jmifile( 390 'framerate_decoded', jmidata), 391 units='fps', higher_is_better=True) 392 393 self.output_perf_value(description='avg_vid_out_framerate_input', 394 value=self._get_average('framerate_outgoing', jmidata), 395 units='fps', higher_is_better=True) 396 397 self.output_perf_value(description='vid_out_framerate_input', 398 value=self._get_data_from_jmifile( 399 'framerate_outgoing', jmidata), 400 units='fps', higher_is_better=True) 401 402 self.output_perf_value(description='std_dev_vid_out_framerate_input', 403 value=self._get_std_dev('framerate_outgoing', jmidata), 404 units='fps', higher_is_better=True) 405 406 self.output_perf_value(description='avg_vid_in_framerate_to_renderer', 407 value=self._get_average('framerate_to_renderer', jmidata), 408 units='fps', higher_is_better=True) 409 410 self.output_perf_value(description='vid_in_framerate_to_renderer', 411 value=self._get_data_from_jmifile( 412 'framerate_to_renderer', jmidata), 413 units='fps', higher_is_better=True) 414 415 self.output_perf_value(description='avg_vid_in_framerate_received', 416 value=self._get_average('framerate_received', jmidata), 417 units='fps', higher_is_better=True) 418 419 self.output_perf_value(description='vid_in_framerate_received', 420 value=self._get_data_from_jmifile( 421 'framerate_received', jmidata), 422 units='fps', higher_is_better=True) 423 424 self.output_perf_value(description='avg_vid_out_framerate_sent', 425 value=self._get_average('framerate_sent', jmidata), 426 units='fps', higher_is_better=True) 427 428 self.output_perf_value(description='vid_out_framerate_sent', 429 value=self._get_data_from_jmifile('framerate_sent', jmidata), 430 units='fps', higher_is_better=True) 431 432 self.output_perf_value(description='std_dev_vid_out_framerate_sent', 433 value=self._get_std_dev('framerate_sent', jmidata), 434 units='fps', higher_is_better=True) 435 436 self.output_perf_value(description='avg_vid_in_frame_width', 437 value=self._get_average('video_received_frame_width', jmidata), 438 units='fps', higher_is_better=True) 439 440 self.output_perf_value(description='vid_in_frame_width', 441 value=self._get_data_from_jmifile( 442 'video_received_frame_width', jmidata), 443 units='fps', higher_is_better=True) 444 445 self.output_perf_value(description='avg_vid_out_frame_width', 446 value=self._get_average('video_sent_frame_width', jmidata), 447 units='fps', higher_is_better=True) 448 449 self.output_perf_value(description='vid_out_frame_width', 450 value=self._get_data_from_jmifile( 451 'video_sent_frame_width', jmidata), 452 units='fps', higher_is_better=True) 453 454 self.output_perf_value(description='avg_vid_out_encode_cpu_usage', 455 value=self._get_average('video_encode_cpu_usage', jmidata), 456 units='percent', higher_is_better=False) 457 458 self.output_perf_value(description='vid_out_encode_cpu_usage', 459 value=self._get_data_from_jmifile( 460 'video_encode_cpu_usage', jmidata), 461 units='percent', higher_is_better=False) 462 463 total_vid_packets_sent = self._get_sum('video_packets_sent', jmidata) 464 total_vid_packets_lost = self._get_sum('video_packets_lost', jmidata) 465 lost_packet_percentage = float(total_vid_packets_lost)*100/ \ 466 float(total_vid_packets_sent) if \ 467 total_vid_packets_sent else 0 468 469 self.output_perf_value(description='lost_packet_percentage', 470 value=lost_packet_percentage, units='percent', 471 higher_is_better=False) 472 473 num_processors = self._get_data_from_jmifile('cpu_processors', jmidata) 474 avg_total_cpu = self._get_average('cpu_percent', jmidata) 475 avg_render_cpu = self._get_average('renderer_cpu_percent', jmidata) 476 total_cpu = self._get_data_from_jmifile('cpu_percent', jmidata) 477 render_cpu = self._get_data_from_jmifile( 478 'renderer_cpu_percent', jmidata) 479 480 cpu_percentage = avg_total_cpu/num_processors if num_processors else 0 481 render_cpu_percent = (avg_render_cpu/num_processors 482 if num_processors else 0) 483 484 cpu_usage = ([value / num_processors for value in total_cpu] 485 if num_processors else 0) 486 render_cpu_usage = ([value / num_processors for value in render_cpu] 487 if num_processors else 0) 488 489 self.output_perf_value(description='avg_cpu_usage_jmi', 490 value=cpu_percentage, 491 units='percent', higher_is_better=False) 492 493 self.output_perf_value(description='cpu_usage_jmi', 494 value=cpu_usage, 495 units='percent', higher_is_better=False) 496 497 self.output_perf_value(description='avg_renderer_cpu_usage', 498 value=render_cpu_percent, 499 units='percent', higher_is_better=False) 500 501 self.output_perf_value(description='renderer_cpu_usage', 502 value=render_cpu_usage, 503 units='percent', higher_is_better=False) 504 505 self.output_perf_value(description='avg_browser_cpu_usage', 506 value=self._get_average('browser_cpu_percent', jmidata), 507 units='percent', higher_is_better=False) 508 509 self.output_perf_value(description='browser_cpu_usage', 510 value=self._get_data_from_jmifile( 511 'browser_cpu_percent', jmidata), 512 units='percent', higher_is_better=False) 513 514 self.output_perf_value(description='avg_gpu_cpu_usage', 515 value=self._get_average('gpu_cpu_percent', jmidata), 516 units='percent', higher_is_better=False) 517 518 self.output_perf_value(description='gpu_cpu_usage', 519 value=self._get_data_from_jmifile( 520 'gpu_cpu_percent', jmidata), 521 units='percent', higher_is_better=False) 522 523 self.output_perf_value(description='avg_active_streams', 524 value=self._get_average('num_active_vid_in_streams', jmidata), 525 units='count', higher_is_better=True) 526 527 self.output_perf_value(description='active_streams', 528 value=self._get_data_from_jmifile( 529 'num_active_vid_in_streams', jmidata), 530 units='count', higher_is_better=True) 531 532 self.output_perf_value(description='std_dev_active_streams', 533 value=self._get_std_dev('num_active_vid_in_streams', jmidata), 534 units='count', higher_is_better=True) 535 536 537 def run_once(self, host=None, is_meeting=False): 538 self.client = host 539 540 factory = remote_facade_factory.RemoteFacadeFactory( 541 host, no_chrome=True) 542 self.system_facade = factory.create_system_facade() 543 self.cfm_facade = factory.create_cfm_facade() 544 545 tpm_utils.ClearTPMOwnerRequest(self.client) 546 547 if self.client.servo: 548 self.client.servo.switch_usbkey('dut') 549 self.client.servo.set('usb_mux_sel3', 'dut_sees_usbkey') 550 time.sleep(_SHORT_TIMEOUT) 551 self.client.servo.set('dut_hub1_rst1', 'off') 552 time.sleep(_SHORT_TIMEOUT) 553 554 try: 555 if is_meeting: 556 self.enroll_device_and_join_meeting() 557 else: 558 self.enroll_device_and_start_hangout() 559 560 self.collect_perf_data() 561 562 if is_meeting: 563 self.cfm_facade.end_meeting_session() 564 else: 565 self.cfm_facade.end_hangout_session() 566 567 self.upload_jmidata() 568 except Exception as e: 569 # Clear tpm to remove device ownership before exiting to ensure 570 # device is not left in an enrolled state. 571 tpm_utils.ClearTPMOwnerRequest(self.client) 572 raise error.TestFail(str(e)) 573 574 tpm_utils.ClearTPMOwnerRequest(self.client) 575 576