1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6# TODO(borenet): This module was copied from build.git and heavily modified to 7# remove dependencies on other modules in build.git. It belongs in a different 8# repo. Remove this once it has been moved. 9 10 11import itertools 12from recipe_engine import recipe_api 13 14 15class IsolateApi(recipe_api.RecipeApi): 16 """APIs for interacting with isolates.""" 17 18 def __init__(self, **kwargs): 19 super(IsolateApi, self).__init__(**kwargs) 20 self._isolate_server = 'https://isolateserver.appspot.com' 21 self._isolated_tests = {} 22 23 @property 24 def isolate_server(self): 25 """URL of Isolate server to use, default is a production one.""" 26 return self._isolate_server 27 28 @isolate_server.setter 29 def isolate_server(self, value): 30 """Changes URL of Isolate server to use.""" 31 self._isolate_server = value 32 33 def clean_isolated_files(self, build_dir): 34 """Cleans out all *.isolated files from the build directory in 35 preparation for the compile. Needed in order to ensure isolates 36 are rebuilt properly because their dependencies are currently not 37 completely described to gyp. 38 """ 39 self.m.python( 40 'clean isolated files', 41 self.resource('find_isolated_tests.py'), 42 [ 43 '--build-dir', build_dir, 44 '--clean-isolated-files' 45 ]) 46 47 def find_isolated_tests(self, build_dir, targets=None, **kwargs): 48 """Returns a step which finds all *.isolated files in a build directory. 49 50 Useful only with 'archive' isolation mode. 51 In 'prepare' mode use 'isolate_tests' instead. 52 53 Assigns the dict {target name -> *.isolated file hash} to the swarm_hashes 54 build property. This implies this step can currently only be run once 55 per recipe. 56 57 If |targets| is None, the step will use all *.isolated files it finds. 58 Otherwise, it will verify that all |targets| are found and will use only 59 them. If some expected targets are missing, will abort the build. 60 """ 61 step_result = self.m.python( 62 'find isolated tests', 63 self.resource('find_isolated_tests.py'), 64 [ 65 '--build-dir', build_dir, 66 '--output-json', self.m.json.output(), 67 ], 68 step_test_data=lambda: (self.test_api.output_json(targets)), 69 **kwargs) 70 71 assert isinstance(step_result.json.output, dict) 72 self._isolated_tests = step_result.json.output 73 if targets is not None and ( 74 step_result.presentation.status != self.m.step.FAILURE): 75 found = set(step_result.json.output) 76 expected = set(targets) 77 if found >= expected: # pragma: no cover 78 # Limit result only to |expected|. 79 self._isolated_tests = { 80 target: step_result.json.output[target] for target in expected 81 } 82 else: 83 # Some expected targets are missing? Fail the step. 84 step_result.presentation.status = self.m.step.FAILURE 85 step_result.presentation.logs['missing.isolates'] = ( 86 ['Failed to find *.isolated files:'] + list(expected - found)) 87 step_result.presentation.properties['swarm_hashes'] = self._isolated_tests 88 # No isolated files found? That looks suspicious, emit warning. 89 if (not self._isolated_tests and 90 step_result.presentation.status != self.m.step.FAILURE): 91 step_result.presentation.status = self.m.step.WARNING 92 93 def isolate_tests(self, build_dir, targets=None, verbose=False, 94 set_swarm_hashes=True, always_use_exparchive=False, 95 **kwargs): 96 """Archives prepared tests in |build_dir| to isolate server. 97 98 src/tools/isolate_driver.py is invoked by ninja during compilation 99 to produce *.isolated.gen.json files that describe how to archive tests. 100 101 This step then uses *.isolated.gen.json files to actually performs the 102 archival. By archiving all tests at once it is able to reduce the total 103 amount of work. Tests share many common files, and such files are processed 104 only once. 105 106 Assigns the dict {target name -> *.isolated file hash} to the swarm_hashes 107 build property (also accessible as 'isolated_tests' property). This implies 108 this step can currently only be run once per recipe. 109 """ 110 # TODO(tansell): Make all steps in this function nested under one overall 111 # 'isolate tests' master step. 112 113 # TODO(vadimsh): Always require |targets| to be passed explicitly. Currently 114 # chromium_trybot, blink_trybot and swarming/canary recipes rely on targets 115 # autodiscovery. The code path in chromium_trybot that needs it is being 116 # deprecated in favor of to *_ng builders, that pass targets explicitly. 117 if targets is None: 118 # Ninja builds <target>.isolated.gen.json files via isolate_driver.py. 119 paths = self.m.file.glob_paths( 120 'find isolated targets', 121 build_dir, 122 '*.isolated.gen.json', 123 test_data=[ 124 'dummy_target_%d.isolated.gen.json' % i 125 for i in (1, 2) 126 ]) 127 targets = [] 128 for p in paths: 129 name = self.m.path.basename(p) 130 assert name.endswith('.isolated.gen.json'), name 131 targets.append(name[:-len('.isolated.gen.json')]) 132 133 # No isolated tests found. 134 if not targets: # pragma: no cover 135 return 136 137 batch_targets = [] 138 exparchive_targets = [] 139 for t in targets: 140 if t.endswith('_exparchive') or always_use_exparchive: 141 exparchive_targets.append(t) 142 else: 143 batch_targets.append(t) 144 145 isolate_steps = [] 146 try: 147 args = [ 148 self.m.swarming_client.path, 149 'exparchive', 150 '--dump-json', self.m.json.output(), 151 '--isolate-server', self._isolate_server, 152 '--eventlog-endpoint', 'prod', 153 ] + (['--verbose'] if verbose else []) 154 155 for target in exparchive_targets: 156 isolate_steps.append( 157 self.m.python( 158 'isolate %s' % target, 159 self.resource('isolate.py'), 160 args + [ 161 '--isolate', build_dir.join('%s.isolate' % target), 162 '--isolated', build_dir.join('%s.isolated' % target), 163 ], 164 step_test_data=lambda: self.test_api.output_json([target]), 165 **kwargs)) 166 167 if batch_targets: 168 # TODO(vadimsh): Differentiate between bad *.isolate and upload errors. 169 # Raise InfraFailure on upload errors. 170 args = [ 171 self.m.swarming_client.path, 172 'batcharchive', 173 '--dump-json', self.m.json.output(), 174 '--isolate-server', self._isolate_server, 175 ] + (['--verbose'] if verbose else []) + [ 176 build_dir.join('%s.isolated.gen.json' % t) for t in batch_targets 177 ] 178 isolate_steps.append( 179 self.m.python( 180 'isolate tests', self.resource('isolate.py'), args, 181 step_test_data=lambda: self.test_api.output_json(batch_targets), 182 **kwargs)) 183 184 # TODO(tansell): Change this to return a dummy "isolate results" or the 185 # top level master step. 186 return isolate_steps[-1] 187 finally: 188 step_result = self.m.step.active_result 189 swarm_hashes = {} 190 for step in isolate_steps: 191 if not step.json.output: 192 continue # pragma: no cover 193 194 for k, v in step.json.output.iteritems(): 195 # TODO(tansell): Raise an error here when it can't clobber an 196 # existing error. This code is currently inside a finally block, 197 # meaning it could be executed when an existing error is occurring. 198 # See https://chromium-review.googlesource.com/c/437024/ 199 #assert k not in swarm_hashes or swarm_hashes[k] == v, ( 200 # "Duplicate hash for target %s was found at step %s." 201 # "Existing hash: %s, New hash: %s") % ( 202 # k, step, swarm_hashes[k], v) 203 swarm_hashes[k] = v 204 205 if swarm_hashes: 206 self._isolated_tests = swarm_hashes 207 208 if set_swarm_hashes: 209 step_result.presentation.properties['swarm_hashes'] = swarm_hashes 210 211 missing = sorted( 212 t for t, h in self._isolated_tests.iteritems() if not h) 213 if missing: 214 step_result.presentation.logs['failed to isolate'] = ( 215 ['Failed to isolate following targets:'] + 216 missing + 217 ['', 'See logs for more information.'] 218 ) 219 for k in missing: 220 self._isolated_tests.pop(k) 221 222 @property 223 def isolated_tests(self): 224 """The dictionary of 'target name -> isolated hash' for this run. 225 226 These come either from the incoming swarm_hashes build property, 227 or from calling find_isolated_tests, above, at some point during the run. 228 """ 229 hashes = self.m.properties.get('swarm_hashes', self._isolated_tests) 230 # Be robust in the case where swarm_hashes is an empty string 231 # instead of an empty dictionary, or similar. 232 if not hashes: 233 return {} # pragma: no covergae 234 return { 235 k.encode('ascii'): v.encode('ascii') 236 for k, v in hashes.iteritems() 237 } 238 239 @property 240 def _run_isolated_path(self): 241 """Returns the path to run_isolated.py.""" 242 return self.m.swarming_client.path.join('run_isolated.py') 243 244 def run_isolated(self, name, isolate_hash, args=None, **kwargs): 245 """Runs an isolated test.""" 246 cmd = [ 247 '--isolated', isolate_hash, 248 '-I', self.isolate_server, 249 '--verbose', 250 ] 251 if args: 252 cmd.append('--') 253 cmd.extend(args) 254 self.m.python(name, self._run_isolated_path, cmd, **kwargs) 255