• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4from datetime import datetime
5import glob
6import optparse
7import os
8import re
9
10import cloud_storage_test_base
11import page_sets
12import pixel_expectations
13
14from telemetry import test
15from telemetry.core import bitmap
16from telemetry.page import cloud_storage
17from telemetry.page import page_test
18
19test_data_dir = os.path.abspath(os.path.join(
20    os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
21
22default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference')
23
24test_harness_script = r"""
25  var domAutomationController = {};
26
27  domAutomationController._succeeded = false;
28  domAutomationController._finished = false;
29
30  domAutomationController.setAutomationId = function(id) {}
31
32  domAutomationController.send = function(msg) {
33    domAutomationController._finished = true;
34
35    if(msg.toLowerCase() == "success") {
36      domAutomationController._succeeded = true;
37    } else {
38      domAutomationController._succeeded = false;
39    }
40  }
41
42  window.domAutomationController = domAutomationController;
43"""
44
45class PixelTestFailure(Exception):
46  pass
47
48def _DidTestSucceed(tab):
49  return tab.EvaluateJavaScript('domAutomationController._succeeded')
50
51class _PixelValidator(cloud_storage_test_base.ValidatorBase):
52  def CustomizeBrowserOptions(self, options):
53    options.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
54
55  def ValidatePage(self, page, tab, results):
56    if not _DidTestSucceed(tab):
57      raise page_test.Failure('Page indicated a failure')
58
59    if not tab.screenshot_supported:
60      raise page_test.Failure('Browser does not support screenshot capture')
61
62    screenshot = tab.Screenshot(5)
63
64    if not screenshot:
65      raise page_test.Failure('Could not capture screenshot')
66
67    if hasattr(page, 'test_rect'):
68      screenshot = screenshot.Crop(
69          page.test_rect[0], page.test_rect[1],
70          page.test_rect[2], page.test_rect[3])
71
72    image_name = self._UrlToImageName(page.display_name)
73
74    if self.options.upload_refimg_to_cloud_storage:
75      if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
76                                                 screenshot):
77        # This is the new reference image; there's nothing to compare against.
78        ref_png = screenshot
79      else:
80        # There was a preexisting reference image, so we might as well
81        # compare against it.
82        ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
83    elif self.options.download_refimg_from_cloud_storage:
84      # This bot doesn't have the ability to properly generate a
85      # reference image, so download it from cloud storage.
86      try:
87        ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
88      except cloud_storage.NotFoundError as e:
89        # There is no reference image yet in cloud storage. This
90        # happens when the revision of the test is incremented or when
91        # a new test is added, because the trybots are not allowed to
92        # produce reference images, only the bots on the main
93        # waterfalls. Report this as a failure so the developer has to
94        # take action by explicitly suppressing the failure and
95        # removing the suppression once the reference images have been
96        # generated. Otherwise silent failures could happen for long
97        # periods of time.
98        raise page_test.Failure('Could not find image %s in cloud storage' %
99                                image_name)
100    else:
101      # Legacy path using on-disk results.
102      ref_png = self._GetReferenceImage(self.options.reference_dir,
103          image_name, page.revision, screenshot)
104
105    # Test new snapshot against existing reference image
106    if not ref_png.IsEqual(screenshot, tolerance=2):
107      if self.options.test_machine_name:
108        self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png)
109      else:
110        self._WriteErrorImages(self.options.generated_dir, image_name,
111                               screenshot, ref_png)
112      raise page_test.Failure('Reference image did not match captured screen')
113
114  def _DeleteOldReferenceImages(self, ref_image_path, cur_revision):
115    if not cur_revision:
116      return
117
118    old_revisions = glob.glob(ref_image_path + "_*.png")
119    for rev_path in old_revisions:
120      m = re.match(r'^.*_(\d+)\.png$', rev_path)
121      if m and int(m.group(1)) < cur_revision:
122        print 'Found deprecated reference image. Deleting rev ' + m.group(1)
123        os.remove(rev_path)
124
125  def _GetReferenceImage(self, img_dir, img_name, cur_revision, screenshot):
126    if not cur_revision:
127      cur_revision = 0
128
129    image_path = os.path.join(img_dir, img_name)
130
131    self._DeleteOldReferenceImages(image_path, cur_revision)
132
133    image_path = image_path + '_' + str(cur_revision) + '.png'
134
135    try:
136      ref_png = bitmap.Bitmap.FromPngFile(image_path)
137    except IOError:
138      ref_png = None
139
140    if ref_png:
141      return ref_png
142
143    print 'Reference image not found. Writing tab contents as reference.'
144
145    self._WriteImage(image_path, screenshot)
146    return screenshot
147
148class Pixel(cloud_storage_test_base.TestBase):
149  test = _PixelValidator
150  page_set = page_sets.PixelTestsPageSet
151
152  @classmethod
153  def AddTestCommandLineArgs(cls, group):
154    super(Pixel, cls).AddTestCommandLineArgs(group)
155    group.add_option('--reference-dir',
156        help='Overrides the default on-disk location for reference images '
157        '(only used for local testing without a cloud storage account)',
158        default=default_reference_image_dir)
159
160  def CreatePageSet(self, options):
161    page_set = super(Pixel, self).CreatePageSet(options)
162    for page in page_set.pages:
163      page.script_to_evaluate_on_commit = test_harness_script
164    return page_set
165
166  def CreateExpectations(self, page_set):
167    return pixel_expectations.PixelExpectations()
168