• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""This is a client side WebGL performance test.
5
6http://hg.mozilla.org/users/bjacob_mozilla.com/webgl-perf-tests/raw-file/3729e8afac99/index.html
7
8From the sources:
9Keep in mind that these tests are not realistic workloads. These are not
10benchmarks aiming to compare browser or GPU performance. These are only useful
11to catch performance regressions in a given browser and system.
12"""
13
14import logging
15import os
16import math
17
18from autotest_lib.client.bin import utils
19from autotest_lib.client.common_lib import error
20from autotest_lib.client.common_lib.cros import chrome
21from autotest_lib.client.cros.graphics import graphics_utils
22
23
24class graphics_WebGLPerformance(graphics_utils.GraphicsTest):
25    """WebGL performance graphics test."""
26    version = 1
27    _test_duration_secs = 0
28    perf_keyval = {}
29    _waived_tests = ['convert-Canvas-to-rgb-float.html',
30                     'convert-Canvas-to-rgb-float-premultiplied.html']
31
32    def setup(self):
33        self.job.setup_dep(['webgl_perf'])
34        self.job.setup_dep(['graphics'])
35
36    def initialize(self):
37        super(graphics_WebGLPerformance, self).initialize()
38
39    def cleanup(self):
40        super(graphics_WebGLPerformance, self).cleanup()
41
42    def run_performance_test(self, browser, test_url):
43        """Runs the performance test from the given url.
44
45        @param browser: The Browser object to run the test with.
46        @param test_url: The URL to the performance test site.
47        """
48        if not utils.wait_for_idle_cpu(60.0, 0.1):
49            if not utils.wait_for_idle_cpu(20.0, 0.2):
50                raise error.TestFail('Failed: Could not get idle CPU.')
51
52        # Kick off test.
53        tab = browser.tabs.New()
54        tab.Navigate(test_url)
55        tab.Activate()
56        tab.WaitForDocumentReadyStateToBeComplete()
57
58        # Wait for test completion.
59        tab.WaitForJavaScriptCondition('test_completed == true',
60                                       timeout=self._test_duration_secs)
61
62        # Get all the result data
63        results = tab.EvaluateJavaScript('testsRun')
64        logging.info('results: %s', results)
65        # Get the geometric mean of individual runtimes.
66        sumOfLogResults = 0
67        sumOfPassed = 0
68        sumOfFailed = 0
69        sumOfWaived = 0
70        for result in results:
71            if result.get('url') in self._waived_tests or result.get('skip'):
72                sumOfWaived += 1
73            elif 'error' in result:
74                self.add_failures(result.get('url'))
75                sumOfFailed += 1
76            else:
77                sumOfLogResults += math.log(result['testResult'])
78                sumOfPassed += 1
79        time_ms_geom_mean = round(100 * math.exp(
80            sumOfLogResults / len(results))) / 100
81
82        logging.info('WebGLPerformance: time_ms_geom_mean = %f',
83                     time_ms_geom_mean)
84
85        # Output numbers for plotting by harness.
86        keyvals = {}
87        keyvals['time_ms_geom_mean'] = time_ms_geom_mean
88        self.write_perf_keyval(keyvals)
89        self.output_perf_value(
90            description='time_geom_mean',
91            value=time_ms_geom_mean,
92            units='ms',
93            higher_is_better=False,
94            graph='time_geom_mean')
95        # Add extra value to the graph distinguishing different boards.
96        variant = utils.get_board_with_frequency_and_memory()
97        desc = 'time_geom_mean-%s' % variant
98        self.output_perf_value(
99            description=desc,
100            value=time_ms_geom_mean,
101            units='ms',
102            higher_is_better=False,
103            graph='time_geom_mean')
104
105        # Get a copy of the test report.
106        test_report = tab.EvaluateJavaScript('test_report')
107        results_path = os.path.join(
108            self.bindir,
109            '../../results/default/graphics_WebGLPerformance/test_report.html')
110        f = open(results_path, 'w+')
111        f.write(test_report)
112        f.close()
113
114        tab.Close()
115        return sumOfPassed, sumOfWaived, sumOfFailed
116
117    def run_once(self, test_duration_secs=2700, fullscreen=True):
118        """Finds a brower with telemetry, and run the test.
119
120        @param test_duration_secs: The test duration in seconds.
121        @param fullscreen: Whether to run the test in fullscreen.
122        """
123        # To avoid 0ms on fast machines like samus the workload was increased.
124        # Unfortunately that makes running on slow machines impractical without
125        # deviating from upstream too much.
126        if utils.get_gpu_family() == 'pinetrail':
127            # TODO(ihf): return a TestPass(message) once available.
128            logging.warning('Test is too slow to run regularly.')
129            return
130
131        self._test_duration_secs = test_duration_secs
132        ext_paths = []
133        if fullscreen:
134            ext_paths.append(
135                os.path.join(self.autodir, 'deps', 'graphics',
136                             'graphics_test_extension'))
137
138        with chrome.Chrome(logged_in=False,
139                           extension_paths=ext_paths,
140                           init_network_controller=True) as cr:
141            websrc_dir = os.path.join(self.autodir, 'deps', 'webgl_perf', 'src')
142            if not cr.browser.platform.SetHTTPServerDirectories(websrc_dir):
143                raise error.TestFail('Failed: Unable to start HTTP server')
144            test_url = cr.browser.platform.http_server.UrlOf(
145                os.path.join(websrc_dir, 'index.html'))
146
147            passed, waived, failed = self.run_performance_test(cr.browser,
148                                                               test_url)
149            logging.debug('Number of tests: %d, passed: %d, '
150                          'waived: %d, failed: %d',
151                          passed + waived + failed, passed, waived, failed)
152            if failed > 0:
153                raise error.TestFail('Failed: %d tests failed.' % failed)
154