1# Copyright 2021 The PDFium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import os 6import logging 7import shlex 8import shutil 9 10from . import pdfium_skia_gold_properties 11from . import pdfium_skia_gold_session_manager 12 13GS_BUCKET = 'skia-pdfium-gm' 14 15 16def _ParseKeyValuePairs(kv_str): 17 """ 18 Parses a string of the type 'key1 value1 key2 value2' into a dict. 19 """ 20 kv_pairs = shlex.split(kv_str) 21 if len(kv_pairs) % 2: 22 raise ValueError('Uneven number of key/value pairs. Got %s' % kv_str) 23 return {kv_pairs[i]: kv_pairs[i + 1] for i in range(0, len(kv_pairs), 2)} 24 25 26def add_skia_gold_args(parser): 27 group = parser.add_argument_group('Skia Gold Arguments') 28 group.add_argument( 29 '--git-revision', help='Revision being tested.', default=None) 30 group.add_argument( 31 '--gerrit-issue', 32 help='For Skia Gold integration. Gerrit issue ID.', 33 default='') 34 group.add_argument( 35 '--gerrit-patchset', 36 help='For Skia Gold integration. Gerrit patch set number.', 37 default='') 38 group.add_argument( 39 '--buildbucket-id', 40 help='For Skia Gold integration. Buildbucket build ID.', 41 default='') 42 group.add_argument( 43 '--bypass-skia-gold-functionality', 44 action='store_true', 45 default=False, 46 help='Bypass all interaction with Skia Gold, effectively disabling the ' 47 'image comparison portion of any tests that use Gold. Only meant to ' 48 'be used in case a Gold outage occurs and cannot be fixed quickly.') 49 local_group = group.add_mutually_exclusive_group() 50 local_group.add_argument( 51 '--local-pixel-tests', 52 action='store_true', 53 default=None, 54 help='Specifies to run the test harness in local run mode or not. When ' 55 'run in local mode, uploading to Gold is disabled and links to ' 56 'help with local debugging are output. Running in local mode also ' 57 'implies --no-luci-auth. If both this and --no-local-pixel-tests are ' 58 'left unset, the test harness will attempt to detect whether it is ' 59 'running on a workstation or not and set this option accordingly.') 60 local_group.add_argument( 61 '--no-local-pixel-tests', 62 action='store_false', 63 dest='local_pixel_tests', 64 help='Specifies to run the test harness in non-local (bot) mode. When ' 65 'run in this mode, data is actually uploaded to Gold and triage links ' 66 'arge generated. If both this and --local-pixel-tests are left unset, ' 67 'the test harness will attempt to detect whether it is running on a ' 68 'workstation or not and set this option accordingly.') 69 group.add_argument( 70 '--no-luci-auth', 71 action='store_true', 72 default=False, 73 help='Don\'t use the service account provided by LUCI for ' 74 'authentication for Skia Gold, instead relying on gsutil to be ' 75 'pre-authenticated. Meant for testing locally instead of on the bots.') 76 77 group.add_argument( 78 '--gold_key', 79 default='', 80 dest="gold_key", 81 help='Key value pairs of config data such like the hardware/software ' 82 'configuration the image was produced on.') 83 group.add_argument( 84 '--gold_output_dir', 85 default='', 86 dest="gold_output_dir", 87 help='Path to the dir where diff output image files are saved, ' 88 'if running locally. If this is a tryjob run, will contain link to skia ' 89 'gold CL triage link. Required with --run-skia-gold.') 90 91 92def clear_gold_output_dir(output_dir): 93 # make sure the output directory exists and is empty. 94 if os.path.exists(output_dir): 95 shutil.rmtree(output_dir, ignore_errors=True) 96 os.makedirs(output_dir) 97 98 99class SkiaGoldTester: 100 101 def __init__(self, source_type, skia_gold_args, process_name=None): 102 """ 103 source_type: source_type (=corpus) field used for all results. 104 skia_gold_args: Parsed arguments from argparse.ArgumentParser. 105 process_name: Unique name of current process, if multiprocessing is on. 106 """ 107 self._source_type = source_type 108 self._output_dir = skia_gold_args.gold_output_dir 109 # goldctl is not thread safe, so each process needs its own directory 110 if process_name: 111 self._output_dir = os.path.join(skia_gold_args.gold_output_dir, 112 process_name) 113 clear_gold_output_dir(self._output_dir) 114 self._keys = _ParseKeyValuePairs(skia_gold_args.gold_key) 115 self._old_gold_props = _ParseKeyValuePairs(skia_gold_args.gold_properties) 116 self._skia_gold_args = skia_gold_args 117 self._skia_gold_session_manager = None 118 self._skia_gold_properties = None 119 120 def WriteCLTriageLink(self, link): 121 # pdfium recipe will read from this file and display the link in the step 122 # presentation 123 assert isinstance(link, str) 124 output_file_name = os.path.join(self._output_dir, 'cl_triage_link.txt') 125 if os.path.exists(output_file_name): 126 os.remove(output_file_name) 127 with open(output_file_name, 'wb') as outfile: 128 outfile.write(link.encode('utf8')) 129 130 def GetSkiaGoldProperties(self): 131 if not self._skia_gold_properties: 132 if self._old_gold_props: 133 self._skia_gold_args.git_revision = self._old_gold_props['gitHash'] 134 self._skia_gold_args.gerrit_issue = self._old_gold_props['issue'] 135 self._skia_gold_args.gerrit_patchset = self._old_gold_props['patchset'] 136 self._skia_gold_args.buildbucket_id = \ 137 self._old_gold_props['buildbucket_build_id'] 138 139 if self._skia_gold_args.local_pixel_tests is None: 140 self._skia_gold_args.local_pixel_tests = 'SWARMING_SERVER' \ 141 not in os.environ 142 143 self._skia_gold_properties = pdfium_skia_gold_properties\ 144 .PDFiumSkiaGoldProperties(self._skia_gold_args) 145 return self._skia_gold_properties 146 147 def GetSkiaGoldSessionManager(self): 148 if not self._skia_gold_session_manager: 149 self._skia_gold_session_manager = pdfium_skia_gold_session_manager\ 150 .PDFiumSkiaGoldSessionManager(self._output_dir, 151 self.GetSkiaGoldProperties()) 152 return self._skia_gold_session_manager 153 154 def IsTryjobRun(self): 155 return self.GetSkiaGoldProperties().IsTryjobRun() 156 157 def GetCLTriageLink(self): 158 return 'https://pdfium-gold.skia.org/search?issue={issue}&crs=gerrit&'\ 159 'corpus={source_type}'.format( 160 issue=self.GetSkiaGoldProperties().issue, source_type=self._source_type) 161 162 def UploadTestResultToSkiaGold(self, image_name, image_path): 163 gold_properties = self.GetSkiaGoldProperties() 164 use_luci = not (gold_properties.local_pixel_tests or 165 gold_properties.no_luci_auth) 166 gold_session = self.GetSkiaGoldSessionManager()\ 167 .GetSkiaGoldSession(self._keys, corpus=self._source_type, 168 bucket=GS_BUCKET) 169 170 status, error = gold_session.RunComparison( 171 name=image_name, png_file=image_path, use_luci=use_luci) 172 173 status_codes =\ 174 self.GetSkiaGoldSessionManager().GetSessionClass().StatusCodes 175 if status == status_codes.SUCCESS: 176 return True 177 if status == status_codes.AUTH_FAILURE: 178 logging.error('Gold authentication failed with output %s', error) 179 elif status == status_codes.INIT_FAILURE: 180 logging.error('Gold initialization failed with output %s', error) 181 elif status == status_codes.COMPARISON_FAILURE_REMOTE: 182 logging.error('Remote comparison failed. See outputted triage links.') 183 elif status == status_codes.COMPARISON_FAILURE_LOCAL: 184 logging.error('Local comparison failed. Local diff files:') 185 _OutputLocalDiffFiles(gold_session, image_name) 186 print() 187 elif status == status_codes.LOCAL_DIFF_FAILURE: 188 logging.error( 189 'Local comparison failed and an error occurred during diff ' 190 'generation: %s', error) 191 # There might be some files, so try outputting them. 192 logging.error('Local diff files:') 193 _OutputLocalDiffFiles(gold_session, image_name) 194 print() 195 else: 196 logging.error( 197 'Given unhandled SkiaGoldSession StatusCode %s with error %s', status, 198 error) 199 200 return False 201 202 203def _OutputLocalDiffFiles(gold_session, image_name): 204 """Logs the local diff image files from the given SkiaGoldSession. 205 206 Args: 207 gold_session: A skia_gold_session.SkiaGoldSession instance to pull files 208 from. 209 image_name: A string containing the name of the image/test that was 210 compared. 211 """ 212 given_file = gold_session.GetGivenImageLink(image_name) 213 closest_file = gold_session.GetClosestImageLink(image_name) 214 diff_file = gold_session.GetDiffImageLink(image_name) 215 failure_message = 'Unable to retrieve link' 216 logging.error('Generated image for %s: %s', image_name, given_file or 217 failure_message) 218 logging.error('Closest image for %s: %s', image_name, closest_file or 219 failure_message) 220 logging.error('Diff image for %s: %s', image_name, diff_file or 221 failure_message) 222