1# Copyright 2013 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Base classes for a test and validator which upload results 6(reference images, error images) to cloud storage.""" 7 8import os 9import re 10import tempfile 11 12from telemetry import test 13from telemetry.core import bitmap 14from telemetry.page import cloud_storage 15from telemetry.page import page_test 16 17test_data_dir = os.path.abspath(os.path.join( 18 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) 19 20default_generated_data_dir = os.path.join(test_data_dir, 'generated') 21 22error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' 23 24def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio): 25 for expectation in expectations: 26 location = expectation["location"] 27 x = int(location[0] * device_pixel_ratio) 28 y = int(location[1] * device_pixel_ratio) 29 30 if x < 0 or y < 0 or x > screenshot.width or y > screenshot.height: 31 raise page_test.Failure( 32 'Expected pixel location [%d, %d] is out of range on [%d, %d] image' % 33 (x, y, screenshot.width, screenshot.height)) 34 35 actual_color = screenshot.GetPixelColor(x, y) 36 expected_color = bitmap.RgbaColor( 37 expectation["color"][0], 38 expectation["color"][1], 39 expectation["color"][2]) 40 if not actual_color.IsEqual(expected_color, expectation["tolerance"]): 41 raise page_test.Failure('Expected pixel at ' + str(location) + 42 ' to be ' + 43 str(expectation["color"]) + " but got [" + 44 str(actual_color.r) + ", " + 45 str(actual_color.g) + ", " + 46 str(actual_color.b) + "]") 47 48class ValidatorBase(page_test.PageTest): 49 def __init__(self): 50 super(ValidatorBase, self).__init__() 51 # Parameters for cloud storage reference images. 52 self.vendor_id = None 53 self.device_id = None 54 self.vendor_string = None 55 self.device_string = None 56 self.msaa = False 57 58 ### 59 ### Routines working with the local disk (only used for local 60 ### testing without a cloud storage account -- the bots do not use 61 ### this code path). 62 ### 63 64 def _UrlToImageName(self, url): 65 image_name = re.sub(r'^(http|https|file)://(/*)', '', url) 66 image_name = re.sub(r'\.\./', '', image_name) 67 image_name = re.sub(r'(\.|/|-)', '_', image_name) 68 return image_name 69 70 def _WriteImage(self, image_path, png_image): 71 output_dir = os.path.dirname(image_path) 72 if not os.path.exists(output_dir): 73 os.makedirs(output_dir) 74 png_image.WritePngFile(image_path) 75 76 def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png): 77 full_image_name = img_name + '_' + str(self.options.build_revision) 78 full_image_name = full_image_name + '.png' 79 80 # Always write the failing image. 81 self._WriteImage( 82 os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot) 83 84 if ref_png: 85 # Save the reference image. 86 # This ensures that we get the right revision number. 87 self._WriteImage( 88 os.path.join(img_dir, full_image_name), ref_png) 89 90 # Save the difference image. 91 diff_png = screenshot.Diff(ref_png) 92 self._WriteImage( 93 os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) 94 95 ### 96 ### Cloud storage code path -- the bots use this. 97 ### 98 99 def _ComputeGpuInfo(self, tab): 100 if ((self.vendor_id and self.device_id) or 101 (self.vendor_string and self.device_string)): 102 return 103 browser = tab.browser 104 if not browser.supports_system_info: 105 raise Exception('System info must be supported by the browser') 106 system_info = browser.GetSystemInfo() 107 if not system_info.gpu: 108 raise Exception('GPU information was absent') 109 device = system_info.gpu.devices[0] 110 if device.vendor_id and device.device_id: 111 self.vendor_id = device.vendor_id 112 self.device_id = device.device_id 113 elif device.vendor_string and device.device_string: 114 self.vendor_string = device.vendor_string 115 self.device_string = device.device_string 116 else: 117 raise Exception('GPU device information was incomplete') 118 self.msaa = not ( 119 'disable_multisampling' in system_info.gpu.driver_bug_workarounds) 120 121 def _FormatGpuInfo(self, tab): 122 self._ComputeGpuInfo(tab) 123 msaa_string = '_msaa' if self.msaa else '_non_msaa' 124 if self.vendor_id: 125 return '%s_%04x_%04x%s' % ( 126 self.options.os_type, self.vendor_id, self.device_id, msaa_string) 127 else: 128 return '%s_%s_%s%s' % ( 129 self.options.os_type, self.vendor_string, self.device_string, 130 msaa_string) 131 132 def _FormatReferenceImageName(self, img_name, page, tab): 133 return '%s_v%s_%s.png' % ( 134 img_name, 135 page.revision, 136 self._FormatGpuInfo(tab)) 137 138 def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False): 139 # This sequence of steps works on all platforms to write a temporary 140 # PNG to disk, following the pattern in bitmap_unittest.py. The key to 141 # avoiding PermissionErrors seems to be to not actually try to write to 142 # the temporary file object, but to re-open its name for all operations. 143 temp_file = tempfile.NamedTemporaryFile().name 144 bitmap.WritePngFile(temp_file) 145 cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public) 146 147 def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot): 148 """Uploads the screenshot to cloud storage as the reference image 149 for this test, unless it already exists. Returns True if the 150 upload was actually performed.""" 151 if not self.options.refimg_cloud_storage_bucket: 152 raise Exception('--refimg-cloud-storage-bucket argument is required') 153 cloud_name = self._FormatReferenceImageName(img_name, page, tab) 154 if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket, 155 cloud_name): 156 self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket, 157 cloud_name, 158 screenshot) 159 return True 160 return False 161 162 def _DownloadFromCloudStorage(self, img_name, page, tab): 163 """Downloads the reference image for the given test from cloud 164 storage, returning it as a Telemetry Bitmap object.""" 165 # TODO(kbr): there's a race condition between the deletion of the 166 # temporary file and gsutil's overwriting it. 167 if not self.options.refimg_cloud_storage_bucket: 168 raise Exception('--refimg-cloud-storage-bucket argument is required') 169 temp_file = tempfile.NamedTemporaryFile().name 170 cloud_storage.Get(self.options.refimg_cloud_storage_bucket, 171 self._FormatReferenceImageName(img_name, page, tab), 172 temp_file) 173 return bitmap.Bitmap.FromPngFile(temp_file) 174 175 def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img): 176 """For a failing run, uploads the failing image, reference image (if 177 supplied), and diff image (if reference image was supplied) to cloud 178 storage. This subsumes the functionality of the 179 archive_gpu_pixel_test_results.py script.""" 180 machine_name = re.sub('\W+', '_', self.options.test_machine_name) 181 upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name) 182 base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir) 183 image_name_with_revision = '%s_%s.png' % ( 184 image_name, self.options.build_revision) 185 self._UploadBitmapToCloudStorage( 186 base_bucket + '/gen', image_name_with_revision, screenshot, 187 public=True) 188 if ref_img: 189 self._UploadBitmapToCloudStorage( 190 base_bucket + '/ref', image_name_with_revision, ref_img, public=True) 191 diff_img = screenshot.Diff(ref_img) 192 self._UploadBitmapToCloudStorage( 193 base_bucket + '/diff', image_name_with_revision, diff_img, 194 public=True) 195 print ('See http://%s.commondatastorage.googleapis.com/' 196 'view_test_results.html?%s for this run\'s test results') % ( 197 error_image_cloud_storage_bucket, upload_dir) 198 199 def _ValidateScreenshotSamples(self, url, 200 screenshot, expectations, device_pixel_ratio): 201 """Samples the given screenshot and verifies pixel color values. 202 The sample locations and expected color values are given in expectations. 203 In case any of the samples do not match the expected color, it raises 204 a Failure and dumps the screenshot locally or cloud storage depending on 205 what machine the test is being run.""" 206 try: 207 _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio) 208 except page_test.Failure: 209 image_name = self._UrlToImageName(url) 210 if self.options.test_machine_name: 211 self._UploadErrorImagesToCloudStorage(image_name, screenshot, None) 212 else: 213 self._WriteErrorImages(self.options.generated_dir, image_name, 214 screenshot, None) 215 raise 216 217 218class TestBase(test.Test): 219 @classmethod 220 def AddTestCommandLineArgs(cls, group): 221 group.add_option('--build-revision', 222 help='Chrome revision being tested.', 223 default="unknownrev") 224 group.add_option('--upload-refimg-to-cloud-storage', 225 dest='upload_refimg_to_cloud_storage', 226 action='store_true', default=False, 227 help='Upload resulting images to cloud storage as reference images') 228 group.add_option('--download-refimg-from-cloud-storage', 229 dest='download_refimg_from_cloud_storage', 230 action='store_true', default=False, 231 help='Download reference images from cloud storage') 232 group.add_option('--refimg-cloud-storage-bucket', 233 help='Name of the cloud storage bucket to use for reference images; ' 234 'required with --upload-refimg-to-cloud-storage and ' 235 '--download-refimg-from-cloud-storage. Example: ' 236 '"chromium-gpu-archive/reference-images"') 237 group.add_option('--os-type', 238 help='Type of operating system on which the pixel test is being run, ' 239 'used only to distinguish different operating systems with the same ' 240 'graphics card. Any value is acceptable, but canonical values are ' 241 '"win", "mac", and "linux", and probably, eventually, "chromeos" ' 242 'and "android").', 243 default='') 244 group.add_option('--test-machine-name', 245 help='Name of the test machine. Specifying this argument causes this ' 246 'script to upload failure images and diffs to cloud storage directly, ' 247 'instead of relying on the archive_gpu_pixel_test_results.py script.', 248 default='') 249 group.add_option('--generated-dir', 250 help='Overrides the default on-disk location for generated test images ' 251 '(only used for local testing without a cloud storage account)', 252 default=default_generated_data_dir) 253