1# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7 8from autotest_lib.client.bin import utils 9from autotest_lib.client.common_lib import error 10from autotest_lib.client.cros import perf 11from autotest_lib.client.cros import service_stopper 12from autotest_lib.client.cros.graphics import graphics_utils 13 14 15class graphics_GLBench(graphics_utils.GraphicsTest): 16 """Run glbench, a benchmark that times graphics intensive activities.""" 17 version = 1 18 preserve_srcdir = True 19 _services = None 20 21 glbench_directory = '/usr/local/glbench/' 22 # Good images. 23 reference_images_file = os.path.join(glbench_directory, 24 'files/glbench_reference_images.txt') 25 # Images that are bad but for which the bug has not been fixed yet. 26 knownbad_images_file = os.path.join(glbench_directory, 27 'files/glbench_knownbad_images.txt') 28 # Images that are bad and for which a fix has been submitted. 29 fixedbad_images_file = os.path.join(glbench_directory, 30 'files/glbench_fixedbad_images.txt') 31 32 # These tests do not draw anything, they can only be used to check 33 # performance. 34 no_checksum_tests = set([ 35 'compositing_no_fill', 36 'pixel_read', 37 'texture_reuse_luminance_teximage2d', 38 'texture_reuse_luminance_texsubimage2d', 39 'texture_reuse_rgba_teximage2d', 40 'texture_reuse_rgba_texsubimage2d', 41 'context_glsimple', 42 'swap_glsimple', 43 ]) 44 45 unit_higher_is_better = { 46 'mbytes_sec': True, 47 'mpixels_sec': True, 48 'mtexel_sec': True, 49 'mtri_sec': True, 50 'mvtx_sec': True, 51 'us': False, 52 '1280x768_fps': True 53 } 54 55 def initialize(self): 56 super(graphics_GLBench, self).initialize() 57 # If UI is running, we must stop it and restore later. 58 self._services = service_stopper.ServiceStopper(['ui']) 59 self._services.stop_services() 60 61 def cleanup(self): 62 if self._services: 63 self._services.restore_services() 64 super(graphics_GLBench, self).cleanup() 65 66 def is_no_checksum_test(self, testname): 67 """Check if given test requires no screenshot checksum. 68 69 @param testname: name of test to check. 70 """ 71 for prefix in self.no_checksum_tests: 72 if testname.startswith(prefix): 73 return True 74 return False 75 76 def load_imagenames(self, filename): 77 """Loads text file with MD5 file names. 78 79 @param filename: name of file to load. 80 """ 81 imagenames = os.path.join(self.autodir, filename) 82 with open(imagenames, 'r') as f: 83 imagenames = f.read() 84 return imagenames 85 86 @graphics_utils.GraphicsTest.failure_report_decorator('graphics_GLBench') 87 def run_once(self, options='', hasty=False): 88 """Run the test. 89 90 @param options: String of options to run the glbench executable with. 91 @param hasty: Run the test more quickly by running fewer iterations, 92 lower resolution, and without waiting for the dut to cool down. 93 """ 94 # Run the test, saving is optional and helps with debugging 95 # and reference image management. If unknown images are 96 # encountered one can take them from the outdir and copy 97 # them (after verification) into the reference image dir. 98 exefile = os.path.join(self.glbench_directory, 'bin/glbench') 99 outdir = self.outputdir 100 options += ' -save -outdir=' + outdir 101 # Using the -hasty option we run only a subset of tests without waiting 102 # for thermals to normalize. Test should complete in 15-20 seconds. 103 if hasty: 104 options += ' -hasty' 105 106 cmd = '%s %s' % (exefile, options) 107 summary = None 108 pc_error_reason = None 109 try: 110 if hasty: 111 # On BVT the test will not monitor thermals so we will not verify its 112 # correct status using PerfControl 113 summary = utils.run(cmd, 114 stderr_is_expected=False, 115 stdout_tee=utils.TEE_TO_LOGS, 116 stderr_tee=utils.TEE_TO_LOGS).stdout 117 else: 118 utils.report_temperature(self, 'temperature_1_start') 119 # Wrap the test run inside of a PerfControl instance to make machine 120 # behavior more consistent. 121 with perf.PerfControl() as pc: 122 if not pc.verify_is_valid(): 123 raise error.TestFail('Failed: %s' % pc.get_error_reason()) 124 utils.report_temperature(self, 'temperature_2_before_test') 125 126 # Run the test. If it gets the CPU too hot pc should notice. 127 summary = utils.run(cmd, 128 stderr_is_expected=False, 129 stdout_tee=utils.TEE_TO_LOGS, 130 stderr_tee=utils.TEE_TO_LOGS).stdout 131 if not pc.verify_is_valid(): 132 # Defer error handling until after perf report. 133 pc_error_reason = pc.get_error_reason() 134 except error.CmdError: 135 raise error.TestFail('Failed: CmdError running %s' % cmd) 136 except error.CmdTimeoutError: 137 raise error.TestFail('Failed: CmdTimeout running %s' % cmd) 138 139 # Write a copy of stdout to help debug failures. 140 results_path = os.path.join(self.outputdir, 'summary.txt') 141 f = open(results_path, 'w+') 142 f.write('# ---------------------------------------------------\n') 143 f.write('# [' + cmd + ']\n') 144 f.write(summary) 145 f.write('\n# -------------------------------------------------\n') 146 f.write('# [graphics_GLBench.py postprocessing]\n') 147 148 # Analyze the output. Sample: 149 ## board_id: NVIDIA Corporation - Quadro FX 380/PCI/SSE2 150 ## Running: ../glbench -save -outdir=img 151 #swap_swap = 221.36 us [swap_swap.pixmd5-20dbc...f9c700d2f.png] 152 results = summary.splitlines() 153 if not results: 154 f.close() 155 raise error.TestFail('Failed: No output from test. Check /tmp/' + 156 'test_that_latest/graphics_GLBench/summary.txt' + 157 ' for details.') 158 159 # The good images, the silenced and the zombie/recurring failures. 160 reference_imagenames = self.load_imagenames(self.reference_images_file) 161 knownbad_imagenames = self.load_imagenames(self.knownbad_images_file) 162 fixedbad_imagenames = self.load_imagenames(self.fixedbad_images_file) 163 164 # Check if we saw GLBench end as expected (without crashing). 165 test_ended_normal = False 166 for line in results: 167 if line.strip().startswith('@TEST_END'): 168 test_ended_normal = True 169 170 # Analyze individual test results in summary. 171 # TODO(pwang): Raise TestFail if an error is detected during glbench. 172 keyvals = {} 173 failed_tests = {} 174 for line in results: 175 if not line.strip().startswith('@RESULT: '): 176 continue 177 keyval, remainder = line[9:].split('[') 178 key, val = keyval.split('=') 179 testname = key.strip() 180 score, unit = val.split() 181 testrating = float(score) 182 imagefile = remainder.split(']')[0] 183 184 if not hasty: 185 higher = self.unit_higher_is_better.get(unit) 186 if higher is None: 187 raise error.TestFail('Failed: Unknown test unit "%s" for %s' % 188 (unit, testname)) 189 # Prepend unit to test name to maintain backwards compatibility with 190 # existing per data. 191 perf_value_name = '%s_%s' % (unit, testname) 192 self.output_perf_value( 193 description=perf_value_name, 194 value=testrating, 195 units=unit, 196 higher_is_better=higher, 197 graph=perf_value_name) 198 199 # Classify result image. 200 if testrating == -1.0: 201 # Tests that generate GL Errors. 202 glerror = imagefile.split('=')[1] 203 f.write('# GLError ' + glerror + ' during test (perf set to -3.0)\n') 204 keyvals[testname] = -3.0 205 failed_tests[testname] = 'GLError' 206 elif testrating == 0.0: 207 # Tests for which glbench does not generate a meaningful perf score. 208 f.write('# No score for test\n') 209 keyvals[testname] = 0.0 210 elif imagefile in fixedbad_imagenames: 211 # We know the image looked bad at some point in time but we thought 212 # it was fixed. Throw an exception as a reminder. 213 keyvals[testname] = -2.0 214 f.write('# fixedbad [' + imagefile + '] (setting perf as -2.0)\n') 215 failed_tests[testname] = imagefile 216 elif imagefile in knownbad_imagenames: 217 # We have triaged the failure and have filed a tracking bug. 218 # Don't throw an exception and remind there is a problem. 219 keyvals[testname] = -1.0 220 f.write('# knownbad [' + imagefile + '] (setting perf as -1.0)\n') 221 # This failure is whitelisted so don't add to failed_tests. 222 elif imagefile in reference_imagenames: 223 # Known good reference images (default). 224 keyvals[testname] = testrating 225 elif imagefile == 'none': 226 # Tests that do not write images can't fail because of them. 227 keyvals[testname] = testrating 228 elif self.is_no_checksum_test(testname): 229 # TODO(ihf): these really should not write any images 230 keyvals[testname] = testrating 231 else: 232 # Completely unknown images. Raise a failure. 233 keyvals[testname] = -2.0 234 failed_tests[testname] = imagefile 235 f.write('# unknown [' + imagefile + '] (setting perf as -2.0)\n') 236 f.close() 237 if not hasty: 238 utils.report_temperature(self, 'temperature_3_after_test') 239 self.write_perf_keyval(keyvals) 240 241 # Raise exception if images don't match. 242 if failed_tests: 243 logging.info('Some images are not matching their reference in %s.', 244 self.reference_images_file) 245 logging.info('Please verify that the output images are correct ' 246 'and if so copy them to the reference directory.') 247 raise error.TestFail('Failed: Some images are not matching their ' 248 'references. Check /tmp/' 249 'test_that_latest/graphics_GLBench/summary.txt' 250 ' for details.') 251 252 if not test_ended_normal: 253 raise error.TestFail( 254 'Failed: No end marker. Presumed crash/missing images.') 255 if pc_error_reason: 256 raise error.TestFail('Failed: %s' % pc_error_reason) 257