• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2021 The PDFium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import os
6import logging
7import shlex
8import shutil
9
10from . import pdfium_skia_gold_properties
11from . import pdfium_skia_gold_session_manager
12
13GS_BUCKET = 'skia-pdfium-gm'
14
15
16def _ParseKeyValuePairs(kv_str):
17  """
18  Parses a string of the type 'key1 value1 key2 value2' into a dict.
19  """
20  kv_pairs = shlex.split(kv_str)
21  if len(kv_pairs) % 2:
22    raise ValueError('Uneven number of key/value pairs. Got %s' % kv_str)
23  return {kv_pairs[i]: kv_pairs[i + 1] for i in range(0, len(kv_pairs), 2)}
24
25
26def add_skia_gold_args(parser):
27  group = parser.add_argument_group('Skia Gold Arguments')
28  group.add_argument(
29      '--git-revision', help='Revision being tested.', default=None)
30  group.add_argument(
31      '--gerrit-issue',
32      help='For Skia Gold integration. Gerrit issue ID.',
33      default='')
34  group.add_argument(
35      '--gerrit-patchset',
36      help='For Skia Gold integration. Gerrit patch set number.',
37      default='')
38  group.add_argument(
39      '--buildbucket-id',
40      help='For Skia Gold integration. Buildbucket build ID.',
41      default='')
42  group.add_argument(
43      '--bypass-skia-gold-functionality',
44      action='store_true',
45      default=False,
46      help='Bypass all interaction with Skia Gold, effectively disabling the '
47      'image comparison portion of any tests that use Gold. Only meant to '
48      'be used in case a Gold outage occurs and cannot be fixed quickly.')
49  local_group = group.add_mutually_exclusive_group()
50  local_group.add_argument(
51      '--local-pixel-tests',
52      action='store_true',
53      default=None,
54      help='Specifies to run the test harness in local run mode or not. When '
55      'run in local mode, uploading to Gold is disabled and links to '
56      'help with local debugging are output. Running in local mode also '
57      'implies --no-luci-auth. If both this and --no-local-pixel-tests are '
58      'left unset, the test harness will attempt to detect whether it is '
59      'running on a workstation or not and set this option accordingly.')
60  local_group.add_argument(
61      '--no-local-pixel-tests',
62      action='store_false',
63      dest='local_pixel_tests',
64      help='Specifies to run the test harness in non-local (bot) mode. When '
65      'run in this mode, data is actually uploaded to Gold and triage links '
66      'arge generated. If both this and --local-pixel-tests are left unset, '
67      'the test harness will attempt to detect whether it is running on a '
68      'workstation or not and set this option accordingly.')
69  group.add_argument(
70      '--no-luci-auth',
71      action='store_true',
72      default=False,
73      help='Don\'t use the service account provided by LUCI for '
74      'authentication for Skia Gold, instead relying on gsutil to be '
75      'pre-authenticated. Meant for testing locally instead of on the bots.')
76
77  group.add_argument(
78      '--gold_key',
79      default='',
80      dest="gold_key",
81      help='Key value pairs of config data such like the hardware/software '
82      'configuration the image was produced on.')
83  group.add_argument(
84      '--gold_output_dir',
85      default='',
86      dest="gold_output_dir",
87      help='Path to the dir where diff output image files are saved, '
88      'if running locally. If this is a tryjob run, will contain link to skia '
89      'gold CL triage link. Required with --run-skia-gold.')
90
91
92def clear_gold_output_dir(output_dir):
93  # make sure the output directory exists and is empty.
94  if os.path.exists(output_dir):
95    shutil.rmtree(output_dir, ignore_errors=True)
96  os.makedirs(output_dir)
97
98
99class SkiaGoldTester:
100
101  def __init__(self, source_type, skia_gold_args, process_name=None):
102    """
103    source_type: source_type (=corpus) field used for all results.
104    skia_gold_args: Parsed arguments from argparse.ArgumentParser.
105    process_name: Unique name of current process, if multiprocessing is on.
106    """
107    self._source_type = source_type
108    self._output_dir = skia_gold_args.gold_output_dir
109    # goldctl is not thread safe, so each process needs its own directory
110    if process_name:
111      self._output_dir = os.path.join(skia_gold_args.gold_output_dir,
112                                      process_name)
113      clear_gold_output_dir(self._output_dir)
114    self._keys = _ParseKeyValuePairs(skia_gold_args.gold_key)
115    self._old_gold_props = _ParseKeyValuePairs(skia_gold_args.gold_properties)
116    self._skia_gold_args = skia_gold_args
117    self._skia_gold_session_manager = None
118    self._skia_gold_properties = None
119
120  def WriteCLTriageLink(self, link):
121    # pdfium recipe will read from this file and display the link in the step
122    # presentation
123    assert isinstance(link, str)
124    output_file_name = os.path.join(self._output_dir, 'cl_triage_link.txt')
125    if os.path.exists(output_file_name):
126      os.remove(output_file_name)
127    with open(output_file_name, 'wb') as outfile:
128      outfile.write(link.encode('utf8'))
129
130  def GetSkiaGoldProperties(self):
131    if not self._skia_gold_properties:
132      if self._old_gold_props:
133        self._skia_gold_args.git_revision = self._old_gold_props['gitHash']
134        self._skia_gold_args.gerrit_issue = self._old_gold_props['issue']
135        self._skia_gold_args.gerrit_patchset = self._old_gold_props['patchset']
136        self._skia_gold_args.buildbucket_id = \
137            self._old_gold_props['buildbucket_build_id']
138
139      if self._skia_gold_args.local_pixel_tests is None:
140        self._skia_gold_args.local_pixel_tests = 'SWARMING_SERVER' \
141            not in os.environ
142
143      self._skia_gold_properties = pdfium_skia_gold_properties\
144          .PDFiumSkiaGoldProperties(self._skia_gold_args)
145    return self._skia_gold_properties
146
147  def GetSkiaGoldSessionManager(self):
148    if not self._skia_gold_session_manager:
149      self._skia_gold_session_manager = pdfium_skia_gold_session_manager\
150          .PDFiumSkiaGoldSessionManager(self._output_dir,
151                                        self.GetSkiaGoldProperties())
152    return self._skia_gold_session_manager
153
154  def IsTryjobRun(self):
155    return self.GetSkiaGoldProperties().IsTryjobRun()
156
157  def GetCLTriageLink(self):
158    return 'https://pdfium-gold.skia.org/search?issue={issue}&crs=gerrit&'\
159    'corpus={source_type}'.format(
160        issue=self.GetSkiaGoldProperties().issue, source_type=self._source_type)
161
162  def UploadTestResultToSkiaGold(self, image_name, image_path):
163    gold_properties = self.GetSkiaGoldProperties()
164    use_luci = not (gold_properties.local_pixel_tests or
165                    gold_properties.no_luci_auth)
166    gold_session = self.GetSkiaGoldSessionManager()\
167        .GetSkiaGoldSession(self._keys, corpus=self._source_type,
168                            bucket=GS_BUCKET)
169
170    status, error = gold_session.RunComparison(
171        name=image_name, png_file=image_path, use_luci=use_luci)
172
173    status_codes =\
174        self.GetSkiaGoldSessionManager().GetSessionClass().StatusCodes
175    if status == status_codes.SUCCESS:
176      return True
177    if status == status_codes.AUTH_FAILURE:
178      logging.error('Gold authentication failed with output %s', error)
179    elif status == status_codes.INIT_FAILURE:
180      logging.error('Gold initialization failed with output %s', error)
181    elif status == status_codes.COMPARISON_FAILURE_REMOTE:
182      logging.error('Remote comparison failed. See outputted triage links.')
183    elif status == status_codes.COMPARISON_FAILURE_LOCAL:
184      logging.error('Local comparison failed. Local diff files:')
185      _OutputLocalDiffFiles(gold_session, image_name)
186      print()
187    elif status == status_codes.LOCAL_DIFF_FAILURE:
188      logging.error(
189          'Local comparison failed and an error occurred during diff '
190          'generation: %s', error)
191      # There might be some files, so try outputting them.
192      logging.error('Local diff files:')
193      _OutputLocalDiffFiles(gold_session, image_name)
194      print()
195    else:
196      logging.error(
197          'Given unhandled SkiaGoldSession StatusCode %s with error %s', status,
198          error)
199
200    return False
201
202
203def _OutputLocalDiffFiles(gold_session, image_name):
204  """Logs the local diff image files from the given SkiaGoldSession.
205
206  Args:
207    gold_session: A skia_gold_session.SkiaGoldSession instance to pull files
208        from.
209    image_name: A string containing the name of the image/test that was
210        compared.
211  """
212  given_file = gold_session.GetGivenImageLink(image_name)
213  closest_file = gold_session.GetClosestImageLink(image_name)
214  diff_file = gold_session.GetDiffImageLink(image_name)
215  failure_message = 'Unable to retrieve link'
216  logging.error('Generated image for %s: %s', image_name, given_file or
217                failure_message)
218  logging.error('Closest image for %s: %s', image_name, closest_file or
219                failure_message)
220  logging.error('Diff image for %s: %s', image_name, diff_file or
221                failure_message)
222