1# Lint as: python2, python3 2# Copyright 2018 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Shared functions by dynamic_suite/suite.py & skylab_suite/cros_suite.py.""" 7 8from __future__ import absolute_import 9from __future__ import division 10from __future__ import print_function 11 12import datetime 13import logging 14import multiprocessing 15import re 16import six 17from six.moves import zip 18 19import common 20 21from autotest_lib.client.common_lib import control_data 22from autotest_lib.client.common_lib import error 23from autotest_lib.client.common_lib import global_config 24from autotest_lib.client.common_lib import time_utils 25from autotest_lib.client.common_lib.cros import dev_server 26from autotest_lib.server.cros import provision 27from autotest_lib.server.cros.dynamic_suite import constants 28from autotest_lib.server.cros.dynamic_suite import control_file_getter 29from autotest_lib.server.cros.dynamic_suite import tools 30 31ENABLE_CONTROLS_IN_BATCH = global_config.global_config.get_config_value( 32 'CROS', 'enable_getting_controls_in_batch', type=bool, default=False) 33 34 35def canonicalize_suite_name(suite_name): 36 """Canonicalize the suite's name. 37 38 @param suite_name: the name of the suite. 39 """ 40 # Do not change this naming convention without updating 41 # site_utils.parse_job_name. 42 return 'test_suites/control.%s' % suite_name 43 44 45def _formatted_now(): 46 """Format the current datetime.""" 47 return datetime.datetime.now().strftime(time_utils.TIME_FMT) 48 49 50def make_builds_from_options(options): 51 """Create a dict of builds for creating a suite job. 52 53 The returned dict maps version label prefixes to build names. Together, 54 each key-value pair describes a complete label. 55 56 @param options: SimpleNamespace from argument parsing. 57 58 @return: dict mapping version label prefixes to build names 59 """ 60 builds = {} 61 build_prefix = None 62 if options.build: 63 build_prefix = provision.get_version_label_prefix(options.build) 64 builds[build_prefix] = options.build 65 66 if options.cheets_build: 67 builds[provision.CROS_ANDROID_VERSION_PREFIX] = options.cheets_build 68 if build_prefix == provision.CROS_VERSION_PREFIX: 69 builds[build_prefix] += provision.CHEETS_SUFFIX 70 71 if options.firmware_rw_build: 72 builds[provision.FW_RW_VERSION_PREFIX] = options.firmware_rw_build 73 74 if options.firmware_ro_build: 75 builds[provision.FW_RO_VERSION_PREFIX] = options.firmware_ro_build 76 77 return builds 78 79 80def get_test_source_build(builds, **dargs): 81 """Get the build of test code. 82 83 Get the test source build from arguments. If parameter 84 `test_source_build` is set and has a value, return its value. Otherwise 85 returns the ChromeOS build name if it exists. If ChromeOS build is not 86 specified either, raise SuiteArgumentException. 87 88 @param builds: the builds on which we're running this suite. It's a 89 dictionary of version_prefix:build. 90 @param **dargs: Any other Suite constructor parameters, as described 91 in Suite.__init__ docstring. 92 93 @return: The build contains the test code. 94 @raise: SuiteArgumentException if both test_source_build and ChromeOS 95 build are not specified. 96 97 """ 98 if dargs.get('test_source_build', None): 99 return dargs['test_source_build'] 100 101 cros_build = builds.get(provision.CROS_VERSION_PREFIX, None) 102 if cros_build.endswith(provision.CHEETS_SUFFIX): 103 test_source_build = re.sub( 104 provision.CHEETS_SUFFIX + '$', '', cros_build) 105 else: 106 test_source_build = cros_build 107 108 if not test_source_build: 109 raise error.SuiteArgumentException( 110 'test_source_build must be specified if CrOS build is not ' 111 'specified.') 112 113 return test_source_build 114 115 116def stage_build_artifacts(build, hostname=None, artifacts=[]): 117 """ 118 Ensure components of |build| necessary for installing images are staged. 119 120 @param build image we want to stage. 121 @param hostname hostname of a dut may run test on. This is to help to locate 122 a devserver closer to duts if needed. Default is None. 123 @param artifacts A list of string artifact name to be staged. 124 125 @raises StageControlFileFailure: if the dev server throws 500 while staging 126 suite control files. 127 128 @return: dev_server.ImageServer instance to use with this build. 129 @return: timings dictionary containing staging start/end times. 130 """ 131 timings = {} 132 # Ensure components of |build| necessary for installing images are staged 133 # on the dev server. However set synchronous to False to allow other 134 # components to be downloaded in the background. 135 ds = dev_server.resolve(build, hostname=hostname) 136 ds_name = ds.hostname 137 timings[constants.DOWNLOAD_STARTED_TIME] = _formatted_now() 138 try: 139 artifacts_to_stage = ['test_suites', 'control_files'] 140 artifacts_to_stage.extend(artifacts if artifacts else []) 141 ds.stage_artifacts(image=build, artifacts=artifacts_to_stage) 142 except dev_server.DevServerException as e: 143 raise error.StageControlFileFailure( 144 "Failed to stage %s on %s: %s" % (build, ds_name, e)) 145 timings[constants.PAYLOAD_FINISHED_TIME] = _formatted_now() 146 return ds, timings 147 148 149def get_control_file_by_build(build, ds, suite_name): 150 """Return control file contents for |suite_name|. 151 152 Query the dev server at |ds| for the control file |suite_name|, included 153 in |build| for |board|. 154 155 @param build: unique name by which to refer to the image from now on. 156 @param ds: a dev_server.DevServer instance to fetch control file with. 157 @param suite_name: canonicalized suite name, e.g. test_suites/control.bvt. 158 @raises ControlFileNotFound if a unique suite control file doesn't exist. 159 @raises NoControlFileList if we can't list the control files at all. 160 @raises ControlFileEmpty if the control file exists on the server, but 161 can't be read. 162 163 @return the contents of the desired control file. 164 """ 165 getter = control_file_getter.DevServerGetter.create(build, ds) 166 devserver_name = ds.hostname 167 # Get the control file for the suite. 168 try: 169 control_file_in = getter.get_control_file_contents_by_name(suite_name) 170 except error.CrosDynamicSuiteException as e: 171 raise type(e)('Failed to get control file for %s ' 172 '(devserver: %s) (error: %s)' % 173 (build, devserver_name, e)) 174 if not control_file_in: 175 raise error.ControlFileEmpty( 176 "Fetching %s returned no data. (devserver: %s)" % 177 (suite_name, devserver_name)) 178 # Force control files to only contain ascii characters. 179 try: 180 control_file_in.encode('ascii') 181 except UnicodeDecodeError as e: 182 raise error.ControlFileMalformed(str(e)) 183 184 return control_file_in 185 186 187def _should_batch_with(cf_getter): 188 """Return whether control files should be fetched in batch. 189 190 This depends on the control file getter and configuration options. 191 192 If cf_getter is a File system ControlFileGetter, the cf_getter will 193 perform a full parse of the root directory associated with the 194 getter. This is the case when it's invoked from suite_preprocessor. 195 196 If cf_getter is a devserver getter, this will look up the suite_name in a 197 suite to control file map generated at build time, and parses the relevant 198 control files alone. This lookup happens on the devserver, so as far 199 as this method is concerned, both cases are equivalent. If 200 enable_controls_in_batch is switched on, this function will call 201 cf_getter.get_suite_info() to get a dict of control files and 202 contents in batch. 203 204 @param cf_getter: a control_file_getter.ControlFileGetter used to list 205 and fetch the content of control files 206 """ 207 return (ENABLE_CONTROLS_IN_BATCH 208 and isinstance(cf_getter, control_file_getter.DevServerGetter)) 209 210 211def _get_cf_texts_for_suite_batched(cf_getter, suite_name): 212 """Get control file content for given suite with batched getter. 213 214 See get_cf_texts_for_suite for params & returns. 215 """ 216 suite_info = cf_getter.get_suite_info(suite_name=suite_name) 217 files = list(suite_info.keys()) 218 filtered_files = _filter_cf_paths(files) 219 for path in filtered_files: 220 yield path, suite_info[path] 221 222 223def _get_cf_texts_for_suite_unbatched(cf_getter, suite_name): 224 """Get control file content for given suite with unbatched getter. 225 226 See get_cf_texts_for_suite for params & returns. 227 """ 228 files = cf_getter.get_control_file_list(suite_name=suite_name) 229 filtered_files = _filter_cf_paths(files) 230 for path in filtered_files: 231 yield path, cf_getter.get_control_file_contents(path) 232 233 234def _filter_cf_paths(paths): 235 """Remove certain control file paths. 236 237 @param paths: Iterable of paths 238 @returns: generator yielding paths 239 """ 240 matcher = re.compile(r'[^/]+/(deps|profilers)/.+') 241 return (path for path in paths if not matcher.match(path)) 242 243 244def get_cf_texts_for_suite(cf_getter, suite_name): 245 """Get control file content for given suite. 246 247 @param cf_getter: A control file getter object, e.g. 248 a control_file_getter.DevServerGetter object. 249 @param suite_name: If specified, this method will attempt to restrain 250 the search space to just this suite's control files. 251 @returns: generator yielding (path, text) tuples 252 """ 253 if _should_batch_with(cf_getter): 254 return _get_cf_texts_for_suite_batched(cf_getter, suite_name) 255 else: 256 return _get_cf_texts_for_suite_unbatched(cf_getter, suite_name) 257 258 259def parse_cf_text(path, text): 260 """Parse control file text. 261 262 @param path: path to control file 263 @param text: control file text contents 264 265 @returns: a ControlData object 266 267 @raises ControlVariableException: There is a syntax error in a 268 control file. 269 """ 270 test = control_data.parse_control_string( 271 text, raise_warnings=True, path=path) 272 test.text = text 273 return test 274 275def parse_cf_text_process(data): 276 """Worker process for parsing control file text 277 278 @param data: Tuple of path, text, forgiving_error, and test_args. 279 280 @returns: Tuple of the path and test ControlData 281 282 @raises ControlVariableException: If forgiving_error is false parsing 283 exceptions are raised instead of logged. 284 """ 285 path, text, forgiving_error, test_args = data 286 287 if test_args: 288 text = tools.inject_vars(test_args, text) 289 290 try: 291 found_test = parse_cf_text(path, text) 292 except control_data.ControlVariableException as e: 293 if not forgiving_error: 294 msg = "Failed parsing %s\n%s" % (path, e) 295 raise control_data.ControlVariableException(msg) 296 logging.warning("Skipping %s\n%s", path, e) 297 except Exception as e: 298 logging.error("Bad %s\n%s", path, e) 299 import traceback 300 logging.error(traceback.format_exc()) 301 else: 302 return (path, found_test) 303 304 305def get_process_limit(): 306 """Limit the number of CPUs to use. 307 308 On a server many autotest instances can run in parallel. Avoid that 309 each of them requests all the CPUs at the same time causing a spike. 310 """ 311 return min(8, multiprocessing.cpu_count()) 312 313 314def parse_cf_text_many(control_file_texts, 315 forgiving_error=False, 316 test_args=None): 317 """Parse control file texts. 318 319 @param control_file_texts: iterable of (path, text) pairs 320 @param test_args: The test args to be injected into test control file. 321 322 @returns: a dictionary of ControlData objects 323 """ 324 tests = {} 325 326 control_file_texts_all = list(control_file_texts) 327 if control_file_texts_all: 328 # Construct input data for worker processes. Each row contains the 329 # path, text, forgiving_error configuration, and test arguments. 330 paths, texts = list(zip(*control_file_texts_all)) 331 worker_data = list(zip(paths, texts, [forgiving_error] * len(paths), 332 [test_args] * len(paths))) 333 pool = multiprocessing.Pool(processes=get_process_limit()) 334 raw_result_list = pool.map(parse_cf_text_process, worker_data) 335 pool.close() 336 pool.join() 337 338 result_list = _current_py_compatible_files(raw_result_list) 339 tests = dict(result_list) 340 341 return tests 342 343 344def _current_py_compatible_files(control_files): 345 """Given a list of control_files, return a list of compatible files. 346 347 Remove blanks/ctrl files with errors (aka not python3 when running 348 python3 compatible) items so the dict conversion doesn't fail. 349 350 @return: List of control files filtered down to those who are compatible 351 with the current running version of python 352 """ 353 result_list = [] 354 for item in control_files: 355 if item: 356 result_list.append(item) 357 elif six.PY2: 358 # Only raise the error in python 2 environments, for now. See 359 # crbug.com/990593 360 raise error.ControlFileMalformed( 361 "Blank or invalid control file. See log for details.") 362 return result_list 363 364 365def retrieve_control_data_for_test(cf_getter, test_name): 366 """Retrieve a test's control file. 367 368 @param cf_getter: a control_file_getter.ControlFileGetter object to 369 list and fetch the control files' content. 370 @param test_name: Name of test to retrieve. 371 372 @raises ControlVariableException: There is a syntax error in a 373 control file. 374 375 @returns a ControlData object 376 """ 377 path = cf_getter.get_control_file_path(test_name) 378 text = cf_getter.get_control_file_contents(path) 379 return parse_cf_text(path, text) 380 381 382def retrieve_for_suite(cf_getter, suite_name='', forgiving_error=False, 383 test_args=None): 384 """Scan through all tests and find all tests. 385 386 @param suite_name: If specified, retrieve this suite's control file. 387 388 @raises ControlVariableException: If forgiving_parser is False and there 389 is a syntax error in a control file. 390 391 @returns a dictionary of ControlData objects that based on given 392 parameters. 393 """ 394 control_file_texts = get_cf_texts_for_suite(cf_getter, suite_name) 395 return parse_cf_text_many(control_file_texts, 396 forgiving_error=forgiving_error, 397 test_args=test_args) 398 399 400def filter_tests(tests, predicate=lambda t: True): 401 """Filter child tests with predicates. 402 403 @tests: A dict of ControlData objects as tests. 404 @predicate: A test filter. By default it's None. 405 406 @returns a list of ControlData objects as tests. 407 """ 408 logging.info('Parsed %s child test control files.', len(tests)) 409 tests = [test for test in six.itervalues(tests) if predicate(test)] 410 tests.sort(key=lambda t: 411 control_data.ControlData.get_test_time_index(t.time), 412 reverse=True) 413 return tests 414 415 416def name_in_tag_predicate(name): 417 """Returns predicate that takes a control file and looks for |name|. 418 419 Builds a predicate that takes in a parsed control file (a ControlData) 420 and returns True if the SUITE tag is present and contains |name|. 421 422 @param name: the suite name to base the predicate on. 423 @return a callable that takes a ControlData and looks for |name| in that 424 ControlData object's suite member. 425 """ 426 return lambda t: name in t.suite_tag_parts 427 428 429def test_name_in_list_predicate(name_list): 430 """Returns a predicate that matches control files by test name. 431 432 The returned predicate returns True for control files whose test name 433 is present in name_list. 434 """ 435 name_set = set(name_list) 436 return lambda t: t.name in name_set 437