xref: /external/autotest/server/site_tests/enterprise_LongevityTrackerServer/enterprise_LongevityTrackerServer.py

# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import csv
import json
import time
import urllib
import urllib2
import logging
import httplib

import enterprise_longevity_helper
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib.cros import tpm_utils
from autotest_lib.server import autotest
from autotest_lib.server import test
from autotest_lib.server.cros.multimedia import remote_facade_factory


STABILIZATION_DURATION = 60
MEASUREMENT_DURATION_SECONDS = 10
TMP_DIRECTORY = '/tmp/'
PERF_FILE_NAME_PREFIX = 'perf'
VERSION_PATTERN = r'^(\d+)\.(\d+)\.(\d+)$'
DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point'
EXPECTED_PARAMS = ['perf_capture_iterations',  'perf_capture_duration',
                   'sample_interval', 'metric_interval', 'test_type',
                   'kiosk_app_attributes']


class PerfUploadingError(Exception):
    """Exception raised in perf_uploader."""
    pass


class enterprise_LongevityTrackerServer(test.test):
    """
    Run Longevity Test: Collect performance data over long duration.

    Run enterprise_KioskEnrollment and clear the TPM as necessary. After
    enterprise enrollment is successful, collect and log cpu, memory, and
    temperature data from the device under test.

    """
    version = 1


    def initialize(self):
        self.temp_dir = os.path.split(self.tmpdir)[0]


    #TODO(krishnargv@): Add a method to retrieve the version of the
    #                   Kiosk app from its manifest.
    def _initialize_test_variables(self):
        """Initialize test variables that will be uploaded to the dashboard."""
        self.board_name = self.system_facade.get_current_board()
        self.chromeos_version = self.system_facade.get_chromeos_release_version()
        epoch_minutes = str(int(time.time() / 60))
        self.point_id = enterprise_longevity_helper.get_point_id(
                self.chromeos_version, epoch_minutes, VERSION_PATTERN)
        self.test_suite_name = self.tagged_testname
        self.perf_capture_duration = self.perf_params['perf_capture_duration']
        self.sample_interval = self.perf_params['sample_interval']
        self.metric_interval = self.perf_params['metric_interval']
        self.perf_results = {'cpu': '0', 'mem': '0', 'temp': '0'}


    def elapsed_time(self, mark_time):
        """
        Get time elapsed since |mark_time|.

        @param mark_time: point in time from which elapsed time is measured.

        @returns time elapsed since the marked time.

        """
        return time.time() - mark_time


    #TODO(krishnargv):  Replace _format_data_for_upload with a call to the
    #                   _format_for_upload method of the perf_uploader.py
    def _format_data_for_upload(self, chart_data):
        """
        Collect chart data into an uploadable data JSON object.

        @param chart_data: performance results formatted as chart data.

        """
        perf_values = {
            'format_version': '1.0',
            'benchmark_name': self.test_suite_name,
            'charts': chart_data,
        }
        #TODO(krishnargv): Add a method to capture the chrome_version.
        dash_entry = {
            'master': 'ChromeOS_Enterprise',
            'bot': 'cros-%s' % self.board_name,
            'point_id': self.point_id,
            'versions': {
                'cros_version': self.chromeos_version,

            },
            'supplemental': {
                'default_rev': 'r_cros_version',
                'kiosk_app_name': 'a_' + self.kiosk_app_name,

            },
            'chart_data': perf_values
        }
        return {'data': json.dumps(dash_entry)}


    #TODO(krishnargv):  Replace _send_to_dashboard with a call to the
    #                   _send_to_dashboard method of the perf_uploader.py
    def _send_to_dashboard(self, data_obj):
        """
        Send formatted perf data to the perf dashboard.

        @param data_obj: data object as returned by _format_data_for_upload().

        @raises PerfUploadingError if an exception was raised when uploading.

        """
        logging.debug('Data_obj to be uploaded: %s', data_obj)
        encoded = urllib.urlencode(data_obj)
        req = urllib2.Request(DASHBOARD_UPLOAD_URL, encoded)
        try:
            urllib2.urlopen(req)
        except urllib2.HTTPError as e:
            raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' %
                                     (e.code, e.msg, data_obj['data']))
        except urllib2.URLError as e:
            raise PerfUploadingError('URLError: %s for JSON %s\n' %
                                     (str(e.reason), data_obj['data']))
        except httplib.HTTPException:
            raise PerfUploadingError('HTTPException for JSON %s\n' %
                                     data_obj['data'])


    def _write_perf_keyvals(self, perf_results):
        """
        Write perf results to keyval file for AutoTest results.

        @param perf_results: dict of attribute performance metrics.

        """
        perf_keyval = {}
        perf_keyval['cpu_usage'] = perf_results['cpu']
        perf_keyval['memory_usage'] = perf_results['mem']
        perf_keyval['temperature'] = perf_results['temp']
        self.write_perf_keyval(perf_keyval)


    def _write_perf_results(self, perf_results):
        """
        Write perf results to results-chart.json file for Perf Dashboard.

        @param perf_results: dict of attribute performance metrics.

        """
        cpu_metric = perf_results['cpu']
        mem_metric = perf_results['mem']
        ec_metric = perf_results['temp']
        self.output_perf_value(description='cpu_usage', value=cpu_metric,
                               units='percent', higher_is_better=False)
        self.output_perf_value(description='mem_usage', value=mem_metric,
                               units='percent', higher_is_better=False)
        self.output_perf_value(description='max_temp', value=ec_metric,
                               units='Celsius', higher_is_better=False)


    def _record_perf_measurements(self, perf_values, perf_writer):
        """
        Record attribute performance measurements, and write to file.

        @param perf_values: dict of attribute performance values.
        @param perf_writer: file to write performance measurements.

        """
        # Get performance measurements.
        cpu_usage = '%.3f' % enterprise_longevity_helper.get_cpu_usage(
                self.system_facade, MEASUREMENT_DURATION_SECONDS)
        mem_usage = '%.3f' % enterprise_longevity_helper.get_memory_usage(
                    self.system_facade)
        max_temp = '%.3f' % enterprise_longevity_helper.get_temperature_data(
                self.client, self.system_facade)

        # Append measurements to attribute lists in perf values dictionary.
        perf_values['cpu'].append(float(cpu_usage))
        perf_values['mem'].append(float(mem_usage))
        perf_values['temp'].append(float(max_temp))

        # Write performance measurements to perf timestamped file.
        time_stamp = time.strftime('%Y/%m/%d %H:%M:%S')
        perf_writer.writerow([time_stamp, cpu_usage, mem_usage, max_temp])
        logging.info('Time: %s, CPU: %r, Mem: %r, Temp: %r',
                     time_stamp, cpu_usage, mem_usage, max_temp)


    def _setup_kiosk_app_on_dut(self, kiosk_app_attributes=None):
        """Enroll the DUT and setup a Kiosk app."""
        info = self.client.host_info_store.get()
        app_config_id = info.get_label_value('app_config_id')
        if app_config_id and app_config_id.startswith(':'):
            app_config_id = app_config_id[1:]
        if kiosk_app_attributes:
            kiosk_app_attributes = kiosk_app_attributes.rstrip()
            self.kiosk_app_name, ext_id = kiosk_app_attributes.split(':')[:2]

        tpm_utils.ClearTPMOwnerRequest(self.client)
        logging.info("Enrolling the DUT to Kiosk mode")
        autotest.Autotest(self.client).run_test(
                'enterprise_KioskEnrollment',
                kiosk_app_attributes=kiosk_app_attributes,
                check_client_result=True)

        #if self.kiosk_app_name == 'riseplayer':
        #    self.kiosk_facade.config_rise_player(ext_id, app_config_id)


    def _run_perf_capture_cycle(self):
        """
        Track performance of Chrome OS over a long period of time.

        This method collects performance measurements, and calculates metrics
        to upload to the performance dashboard. It creates two files to
        collect and store performance values and results: perf_<timestamp>.csv
        and perf_aggregated.csv.

        At the start, it creates a unique perf timestamped file in the test's
        temp_dir. As the cycle runs, it saves a time-stamped performance
        value after each sample interval. Periodically, it calculates
        the 90th percentile performance metrics from these values.

        The perf_<timestamp> files on the device will survive multiple runs
        of the longevity_Tracker by the server-side test, and will also
        survive multiple runs of the server-side test.

        At the end, it opens the perf aggregated file in the test's temp_dir,
        and appends the contents of the perf timestamped file. It then
        copies the perf aggregated file to the results directory as perf.csv.
        This perf.csv file will be consumed by the AutoTest backend when the
        server-side test ends.

        Note that the perf_aggregated.csv file will grow larger with each run
        of longevity_Tracker on the device by the server-side test.

        This method will capture perf metrics every SAMPLE_INTERVAL secs, at
        each METRIC_INTERVAL the 90 percentile of the collected metrics is
        calculated and saved. The perf capture runs for PERF_CAPTURE_DURATION
        secs. At the end of the PERF_CAPTURE_DURATION time interval the median
        value of all 90th percentile metrics is returned.

        @returns list of median performance metrics.

        """
        test_start_time = time.time()

        perf_values = {'cpu': [], 'mem': [], 'temp': []}
        perf_metrics = {'cpu': [], 'mem': [], 'temp': []}

         # Create perf_<timestamp> file and writer.
        timestamp_fname = (PERF_FILE_NAME_PREFIX +
                           time.strftime('_%Y-%m-%d_%H-%M') + '.csv')
        timestamp_fpath = os.path.join(self.temp_dir, timestamp_fname)
        timestamp_file = enterprise_longevity_helper.open_perf_file(
                timestamp_fpath)
        timestamp_writer = csv.writer(timestamp_file)

        # Align time of loop start with the sample interval.
        test_elapsed_time = self.elapsed_time(test_start_time)
        time.sleep(enterprise_longevity_helper.syncup_time(
                test_elapsed_time, self.sample_interval))
        test_elapsed_time = self.elapsed_time(test_start_time)

        metric_start_time = time.time()
        metric_prev_time = metric_start_time

        metric_elapsed_prev_time = self.elapsed_time(metric_prev_time)
        offset = enterprise_longevity_helper.modulo_time(
                metric_elapsed_prev_time, self.metric_interval)
        metric_timer = metric_elapsed_prev_time + offset

        while self.elapsed_time(test_start_time) <= self.perf_capture_duration:
            self._record_perf_measurements(perf_values, timestamp_writer)

            # Periodically calculate and record 90th percentile metrics.
            metric_elapsed_prev_time = self.elapsed_time(metric_prev_time)
            metric_timer = metric_elapsed_prev_time + offset
            if metric_timer >= self.metric_interval:
                enterprise_longevity_helper.record_90th_metrics(
                        perf_values, perf_metrics)
                perf_values = {'cpu': [], 'mem': [], 'temp': []}

            # Set previous time to current time.
                metric_prev_time = time.time()
                metric_elapsed_prev_time = self.elapsed_time(metric_prev_time)

                metric_elapsed_time = self.elapsed_time(metric_start_time)
                offset = enterprise_longevity_helper.modulo_time(
                    metric_elapsed_time, self.metric_interval)

                # Set the timer to time elapsed plus offset to next interval.
                metric_timer = metric_elapsed_prev_time + offset

            # Sync the loop time to the sample interval.
            test_elapsed_time = self.elapsed_time(test_start_time)
            time.sleep(enterprise_longevity_helper.syncup_time(
                    test_elapsed_time, self.sample_interval))

        # Close perf timestamp file.
        timestamp_file.close()

         # Open perf timestamp file to read, and aggregated file to append.
        timestamp_file = open(timestamp_fpath, 'r')
        aggregated_fname = (PERF_FILE_NAME_PREFIX + '_aggregated.csv')
        aggregated_fpath = os.path.join(self.temp_dir, aggregated_fname)
        aggregated_file = enterprise_longevity_helper.open_perf_file(
                aggregated_fpath)

         # Append contents of perf timestamp file to perf aggregated file.
        enterprise_longevity_helper.append_to_aggregated_file(
                timestamp_file, aggregated_file)
        timestamp_file.close()
        aggregated_file.close()

        # Copy perf aggregated file to test results directory.
        enterprise_longevity_helper.copy_aggregated_to_resultsdir(
                self.resultsdir, aggregated_fpath, 'perf.csv')

        # Return median of each attribute performance metric.
        logging.info("Perf_metrics: %r ", perf_metrics)
        return enterprise_longevity_helper.get_median_metrics(perf_metrics)


    def run_once(self, host=None, perf_params=None):
        self.client = host
        self.kiosk_app_name = None
        self.perf_params = perf_params
        logging.info('Perf params: %r', self.perf_params)

        if not enterprise_longevity_helper.verify_perf_params(
                EXPECTED_PARAMS, self.perf_params):
            raise error.TestFail('Missing or incorrect perf_params in the'
                                 ' control file. Refer to the README.txt for'
                                 ' info on perf params.: %r'
                                  %(self.perf_params))

        factory = remote_facade_factory.RemoteFacadeFactory(
                host, no_chrome=True)
        self.system_facade = factory.create_system_facade()
        self.kiosk_facade = factory.create_kiosk_facade()

        self._setup_kiosk_app_on_dut(self.perf_params['kiosk_app_attributes'])
        time.sleep(STABILIZATION_DURATION)

        self._initialize_test_variables()
        for iteration in range(self.perf_params['perf_capture_iterations']):
            #TODO(krishnargv@): Add a method to verify that the Kiosk app is
            #                   active and is running on the DUT.
            logging.info("Running perf_capture Iteration: %d", iteration+1)
            self.perf_results = self._run_perf_capture_cycle()
            self._write_perf_keyvals(self.perf_results)
            self._write_perf_results(self.perf_results)

            # Post perf results directly to performance dashboard. You may view
            # uploaded data at https://chromeperf.appspot.com/new_points,
            # with test path pattern=ChromeOS_Enterprise/cros-*/longevity*/*
            if perf_params['test_type'] == 'multiple_samples':
                chart_data = enterprise_longevity_helper.read_perf_results(
                        self.resultsdir, 'results-chart.json')
                data_obj = self._format_data_for_upload(chart_data)
                self._send_to_dashboard(data_obj)
        tpm_utils.ClearTPMOwnerRequest(self.client)