# Copyright 2017 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import os import csv import json import time import urllib import urllib2 import logging import httplib import enterprise_longevity_helper from autotest_lib.client.common_lib import error from autotest_lib.client.common_lib.cros import tpm_utils from autotest_lib.server import autotest from autotest_lib.server import test from autotest_lib.server.cros.multimedia import remote_facade_factory STABILIZATION_DURATION = 60 MEASUREMENT_DURATION_SECONDS = 10 TMP_DIRECTORY = '/tmp/' PERF_FILE_NAME_PREFIX = 'perf' VERSION_PATTERN = r'^(\d+)\.(\d+)\.(\d+)$' DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point' EXPECTED_PARAMS = ['perf_capture_iterations', 'perf_capture_duration', 'sample_interval', 'metric_interval', 'test_type', 'kiosk_app_attributes'] class PerfUploadingError(Exception): """Exception raised in perf_uploader.""" pass class enterprise_LongevityTrackerServer(test.test): """ Run Longevity Test: Collect performance data over long duration. Run enterprise_KioskEnrollment and clear the TPM as necessary. After enterprise enrollment is successful, collect and log cpu, memory, and temperature data from the device under test. """ version = 1 def initialize(self): self.temp_dir = os.path.split(self.tmpdir)[0] #TODO(krishnargv@): Add a method to retrieve the version of the # Kiosk app from its manifest. def _initialize_test_variables(self): """Initialize test variables that will be uploaded to the dashboard.""" self.board_name = self.system_facade.get_current_board() self.chromeos_version = self.system_facade.get_chromeos_release_version() epoch_minutes = str(int(time.time() / 60)) self.point_id = enterprise_longevity_helper.get_point_id( self.chromeos_version, epoch_minutes, VERSION_PATTERN) self.test_suite_name = self.tagged_testname self.perf_capture_duration = self.perf_params['perf_capture_duration'] self.sample_interval = self.perf_params['sample_interval'] self.metric_interval = self.perf_params['metric_interval'] self.perf_results = {'cpu': '0', 'mem': '0', 'temp': '0'} def elapsed_time(self, mark_time): """ Get time elapsed since |mark_time|. @param mark_time: point in time from which elapsed time is measured. @returns time elapsed since the marked time. """ return time.time() - mark_time #TODO(krishnargv): Replace _format_data_for_upload with a call to the # _format_for_upload method of the perf_uploader.py def _format_data_for_upload(self, chart_data): """ Collect chart data into an uploadable data JSON object. @param chart_data: performance results formatted as chart data. """ perf_values = { 'format_version': '1.0', 'benchmark_name': self.test_suite_name, 'charts': chart_data, } #TODO(krishnargv): Add a method to capture the chrome_version. dash_entry = { 'master': 'ChromeOS_Enterprise', 'bot': 'cros-%s' % self.board_name, 'point_id': self.point_id, 'versions': { 'cros_version': self.chromeos_version, }, 'supplemental': { 'default_rev': 'r_cros_version', 'kiosk_app_name': 'a_' + self.kiosk_app_name, }, 'chart_data': perf_values } return {'data': json.dumps(dash_entry)} #TODO(krishnargv): Replace _send_to_dashboard with a call to the # _send_to_dashboard method of the perf_uploader.py def _send_to_dashboard(self, data_obj): """ Send formatted perf data to the perf dashboard. @param data_obj: data object as returned by _format_data_for_upload(). @raises PerfUploadingError if an exception was raised when uploading. """ logging.debug('Data_obj to be uploaded: %s', data_obj) encoded = urllib.urlencode(data_obj) req = urllib2.Request(DASHBOARD_UPLOAD_URL, encoded) try: urllib2.urlopen(req) except urllib2.HTTPError as e: raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' % (e.code, e.msg, data_obj['data'])) except urllib2.URLError as e: raise PerfUploadingError('URLError: %s for JSON %s\n' % (str(e.reason), data_obj['data'])) except httplib.HTTPException: raise PerfUploadingError('HTTPException for JSON %s\n' % data_obj['data']) def _write_perf_keyvals(self, perf_results): """ Write perf results to keyval file for AutoTest results. @param perf_results: dict of attribute performance metrics. """ perf_keyval = {} perf_keyval['cpu_usage'] = perf_results['cpu'] perf_keyval['memory_usage'] = perf_results['mem'] perf_keyval['temperature'] = perf_results['temp'] self.write_perf_keyval(perf_keyval) def _write_perf_results(self, perf_results): """ Write perf results to results-chart.json file for Perf Dashboard. @param perf_results: dict of attribute performance metrics. """ cpu_metric = perf_results['cpu'] mem_metric = perf_results['mem'] ec_metric = perf_results['temp'] self.output_perf_value(description='cpu_usage', value=cpu_metric, units='percent', higher_is_better=False) self.output_perf_value(description='mem_usage', value=mem_metric, units='percent', higher_is_better=False) self.output_perf_value(description='max_temp', value=ec_metric, units='Celsius', higher_is_better=False) def _record_perf_measurements(self, perf_values, perf_writer): """ Record attribute performance measurements, and write to file. @param perf_values: dict of attribute performance values. @param perf_writer: file to write performance measurements. """ # Get performance measurements. cpu_usage = '%.3f' % enterprise_longevity_helper.get_cpu_usage( self.system_facade, MEASUREMENT_DURATION_SECONDS) mem_usage = '%.3f' % enterprise_longevity_helper.get_memory_usage( self.system_facade) max_temp = '%.3f' % enterprise_longevity_helper.get_temperature_data( self.client, self.system_facade) # Append measurements to attribute lists in perf values dictionary. perf_values['cpu'].append(float(cpu_usage)) perf_values['mem'].append(float(mem_usage)) perf_values['temp'].append(float(max_temp)) # Write performance measurements to perf timestamped file. time_stamp = time.strftime('%Y/%m/%d %H:%M:%S') perf_writer.writerow([time_stamp, cpu_usage, mem_usage, max_temp]) logging.info('Time: %s, CPU: %r, Mem: %r, Temp: %r', time_stamp, cpu_usage, mem_usage, max_temp) def _setup_kiosk_app_on_dut(self, kiosk_app_attributes=None): """Enroll the DUT and setup a Kiosk app.""" info = self.client.host_info_store.get() app_config_id = info.get_label_value('app_config_id') if app_config_id and app_config_id.startswith(':'): app_config_id = app_config_id[1:] if kiosk_app_attributes: kiosk_app_attributes = kiosk_app_attributes.rstrip() self.kiosk_app_name, ext_id = kiosk_app_attributes.split(':')[:2] tpm_utils.ClearTPMOwnerRequest(self.client) logging.info("Enrolling the DUT to Kiosk mode") autotest.Autotest(self.client).run_test( 'enterprise_KioskEnrollment', kiosk_app_attributes=kiosk_app_attributes, check_client_result=True) #if self.kiosk_app_name == 'riseplayer': # self.kiosk_facade.config_rise_player(ext_id, app_config_id) def _run_perf_capture_cycle(self): """ Track performance of Chrome OS over a long period of time. This method collects performance measurements, and calculates metrics to upload to the performance dashboard. It creates two files to collect and store performance values and results: perf_.csv and perf_aggregated.csv. At the start, it creates a unique perf timestamped file in the test's temp_dir. As the cycle runs, it saves a time-stamped performance value after each sample interval. Periodically, it calculates the 90th percentile performance metrics from these values. The perf_ files on the device will survive multiple runs of the longevity_Tracker by the server-side test, and will also survive multiple runs of the server-side test. At the end, it opens the perf aggregated file in the test's temp_dir, and appends the contents of the perf timestamped file. It then copies the perf aggregated file to the results directory as perf.csv. This perf.csv file will be consumed by the AutoTest backend when the server-side test ends. Note that the perf_aggregated.csv file will grow larger with each run of longevity_Tracker on the device by the server-side test. This method will capture perf metrics every SAMPLE_INTERVAL secs, at each METRIC_INTERVAL the 90 percentile of the collected metrics is calculated and saved. The perf capture runs for PERF_CAPTURE_DURATION secs. At the end of the PERF_CAPTURE_DURATION time interval the median value of all 90th percentile metrics is returned. @returns list of median performance metrics. """ test_start_time = time.time() perf_values = {'cpu': [], 'mem': [], 'temp': []} perf_metrics = {'cpu': [], 'mem': [], 'temp': []} # Create perf_ file and writer. timestamp_fname = (PERF_FILE_NAME_PREFIX + time.strftime('_%Y-%m-%d_%H-%M') + '.csv') timestamp_fpath = os.path.join(self.temp_dir, timestamp_fname) timestamp_file = enterprise_longevity_helper.open_perf_file( timestamp_fpath) timestamp_writer = csv.writer(timestamp_file) # Align time of loop start with the sample interval. test_elapsed_time = self.elapsed_time(test_start_time) time.sleep(enterprise_longevity_helper.syncup_time( test_elapsed_time, self.sample_interval)) test_elapsed_time = self.elapsed_time(test_start_time) metric_start_time = time.time() metric_prev_time = metric_start_time metric_elapsed_prev_time = self.elapsed_time(metric_prev_time) offset = enterprise_longevity_helper.modulo_time( metric_elapsed_prev_time, self.metric_interval) metric_timer = metric_elapsed_prev_time + offset while self.elapsed_time(test_start_time) <= self.perf_capture_duration: self._record_perf_measurements(perf_values, timestamp_writer) # Periodically calculate and record 90th percentile metrics. metric_elapsed_prev_time = self.elapsed_time(metric_prev_time) metric_timer = metric_elapsed_prev_time + offset if metric_timer >= self.metric_interval: enterprise_longevity_helper.record_90th_metrics( perf_values, perf_metrics) perf_values = {'cpu': [], 'mem': [], 'temp': []} # Set previous time to current time. metric_prev_time = time.time() metric_elapsed_prev_time = self.elapsed_time(metric_prev_time) metric_elapsed_time = self.elapsed_time(metric_start_time) offset = enterprise_longevity_helper.modulo_time( metric_elapsed_time, self.metric_interval) # Set the timer to time elapsed plus offset to next interval. metric_timer = metric_elapsed_prev_time + offset # Sync the loop time to the sample interval. test_elapsed_time = self.elapsed_time(test_start_time) time.sleep(enterprise_longevity_helper.syncup_time( test_elapsed_time, self.sample_interval)) # Close perf timestamp file. timestamp_file.close() # Open perf timestamp file to read, and aggregated file to append. timestamp_file = open(timestamp_fpath, 'r') aggregated_fname = (PERF_FILE_NAME_PREFIX + '_aggregated.csv') aggregated_fpath = os.path.join(self.temp_dir, aggregated_fname) aggregated_file = enterprise_longevity_helper.open_perf_file( aggregated_fpath) # Append contents of perf timestamp file to perf aggregated file. enterprise_longevity_helper.append_to_aggregated_file( timestamp_file, aggregated_file) timestamp_file.close() aggregated_file.close() # Copy perf aggregated file to test results directory. enterprise_longevity_helper.copy_aggregated_to_resultsdir( self.resultsdir, aggregated_fpath, 'perf.csv') # Return median of each attribute performance metric. logging.info("Perf_metrics: %r ", perf_metrics) return enterprise_longevity_helper.get_median_metrics(perf_metrics) def run_once(self, host=None, perf_params=None): self.client = host self.kiosk_app_name = None self.perf_params = perf_params logging.info('Perf params: %r', self.perf_params) if not enterprise_longevity_helper.verify_perf_params( EXPECTED_PARAMS, self.perf_params): raise error.TestFail('Missing or incorrect perf_params in the' ' control file. Refer to the README.txt for' ' info on perf params.: %r' %(self.perf_params)) factory = remote_facade_factory.RemoteFacadeFactory( host, no_chrome=True) self.system_facade = factory.create_system_facade() self.kiosk_facade = factory.create_kiosk_facade() self._setup_kiosk_app_on_dut(self.perf_params['kiosk_app_attributes']) time.sleep(STABILIZATION_DURATION) self._initialize_test_variables() for iteration in range(self.perf_params['perf_capture_iterations']): #TODO(krishnargv@): Add a method to verify that the Kiosk app is # active and is running on the DUT. logging.info("Running perf_capture Iteration: %d", iteration+1) self.perf_results = self._run_perf_capture_cycle() self._write_perf_keyvals(self.perf_results) self._write_perf_results(self.perf_results) # Post perf results directly to performance dashboard. You may view # uploaded data at https://chromeperf.appspot.com/new_points, # with test path pattern=ChromeOS_Enterprise/cros-*/longevity*/* if perf_params['test_type'] == 'multiple_samples': chart_data = enterprise_longevity_helper.read_perf_results( self.resultsdir, 'results-chart.json') data_obj = self._format_data_for_upload(chart_data) self._send_to_dashboard(data_obj) tpm_utils.ClearTPMOwnerRequest(self.client)