1# Lint as: python2, python3 2# pylint: disable-msg=C0111 3# Copyright 2008 Google Inc. Released under the GPL v2 4 5from __future__ import absolute_import 6from __future__ import division 7from __future__ import print_function 8 9import ast 10import logging 11import textwrap 12import re 13import six 14 15from autotest_lib.client.common_lib import autotest_enum 16from autotest_lib.client.common_lib import global_config 17from autotest_lib.client.common_lib import priorities 18 19 20REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type']) 21OBSOLETE_VARS = set(['experimental']) 22 23CONTROL_TYPE = autotest_enum.AutotestEnum('Server', 'Client', start_value=1) 24CONTROL_TYPE_NAMES = autotest_enum.AutotestEnum(*CONTROL_TYPE.names, 25 string_values=True) 26 27_SUITE_ATTRIBUTE_PREFIX = 'suite:' 28 29CONFIG = global_config.global_config 30 31# Default maximum test result size in kB. 32DEFAULT_MAX_RESULT_SIZE_KB = CONFIG.get_config_value( 33 'AUTOSERV', 'default_max_result_size_KB', type=int, default=20000) 34 35 36class ControlVariableException(Exception): 37 pass 38 39def _validate_control_file_fields(control_file_path, control_file_vars, 40 raise_warnings): 41 """Validate the given set of variables from a control file. 42 43 @param control_file_path: string path of the control file these were 44 loaded from. 45 @param control_file_vars: dict of variables set in a control file. 46 @param raise_warnings: True iff we should raise on invalid variables. 47 48 """ 49 diff = REQUIRED_VARS - set(control_file_vars) 50 if diff: 51 warning = ('WARNING: Not all required control ' 52 'variables were specified in %s. Please define ' 53 '%s.') % (control_file_path, ', '.join(diff)) 54 if raise_warnings: 55 raise ControlVariableException(warning) 56 print(textwrap.wrap(warning, 80)) 57 58 obsolete = OBSOLETE_VARS & set(control_file_vars) 59 if obsolete: 60 warning = ('WARNING: Obsolete variables were ' 61 'specified in %s. Please remove ' 62 '%s.') % (control_file_path, ', '.join(obsolete)) 63 if raise_warnings: 64 raise ControlVariableException(warning) 65 print(textwrap.wrap(warning, 80)) 66 67 68class ControlData(object): 69 # Available TIME settings in control file, the list must be in lower case 70 # and in ascending order, test running faster comes first. 71 TEST_TIME_LIST = ['fast', 'short', 'medium', 'long', 'lengthy'] 72 TEST_TIME = autotest_enum.AutotestEnum(*TEST_TIME_LIST, 73 string_values=False) 74 75 @staticmethod 76 def get_test_time_index(time): 77 """ 78 Get the order of estimated test time, based on the TIME setting in 79 Control file. Faster test gets a lower index number. 80 """ 81 try: 82 return ControlData.TEST_TIME.get_value(time.lower()) 83 except AttributeError: 84 # Raise exception if time value is not a valid TIME setting. 85 error_msg = '%s is not a valid TIME.' % time 86 logging.error(error_msg) 87 raise ControlVariableException(error_msg) 88 89 90 def __init__(self, vars, path, raise_warnings=False): 91 # Defaults 92 self.path = path 93 self.dependencies = set() 94 # TODO(jrbarnette): This should be removed once outside 95 # code that uses can be changed. 96 self.experimental = False 97 self.run_verify = True 98 self.sync_count = 1 99 self.test_parameters = set() 100 self.test_category = '' 101 self.test_class = '' 102 self.job_retries = 0 103 # Default to require server-side package. Unless require_ssp is 104 # explicitly set to False, server-side package will be used for the 105 # job. 106 self.require_ssp = None 107 self.attributes = set() 108 self.max_result_size_KB = DEFAULT_MAX_RESULT_SIZE_KB 109 self.priority = priorities.Priority.DEFAULT 110 self.fast = False 111 112 _validate_control_file_fields(self.path, vars, raise_warnings) 113 114 for key, val in six.iteritems(vars): 115 try: 116 self.set_attr(key, val, raise_warnings) 117 except Exception as e: 118 if raise_warnings: 119 raise 120 print('WARNING: %s; skipping' % e) 121 122 self._patch_up_suites_from_attributes() 123 124 125 @property 126 def suite_tag_parts(self): 127 """Return the part strings of the test's suite tag.""" 128 if hasattr(self, 'suite'): 129 return [part.strip() for part in self.suite.split(',')] 130 else: 131 return [] 132 133 134 def set_attr(self, attr, val, raise_warnings=False): 135 attr = attr.lower() 136 try: 137 set_fn = getattr(self, 'set_%s' % attr) 138 set_fn(val) 139 except AttributeError: 140 # This must not be a variable we care about 141 pass 142 143 144 def _patch_up_suites_from_attributes(self): 145 """Patch up the set of suites this test is part of. 146 147 Legacy builds will not have an appropriate ATTRIBUTES field set. 148 Take the union of suites specified via ATTRIBUTES and suites specified 149 via SUITE. 150 151 SUITE used to be its own variable, but now suites are taken only from 152 the attributes. 153 154 """ 155 156 suite_names = set() 157 # Extract any suites we know ourselves to be in based on the SUITE 158 # line. This line is deprecated, but control files in old builds will 159 # still have it. 160 if hasattr(self, 'suite'): 161 existing_suites = self.suite.split(',') 162 existing_suites = [name.strip() for name in existing_suites] 163 existing_suites = [name for name in existing_suites if name] 164 suite_names.update(existing_suites) 165 166 # Figure out if our attributes mention any suites. 167 for attribute in self.attributes: 168 if not attribute.startswith(_SUITE_ATTRIBUTE_PREFIX): 169 continue 170 suite_name = attribute[len(_SUITE_ATTRIBUTE_PREFIX):] 171 suite_names.add(suite_name) 172 173 # Rebuild the suite field if necessary. 174 if suite_names: 175 self.set_suite(','.join(sorted(list(suite_names)))) 176 177 178 def _set_string(self, attr, val): 179 val = str(val) 180 setattr(self, attr, val) 181 182 183 def _set_option(self, attr, val, options): 184 val = str(val) 185 if val.lower() not in [x.lower() for x in options]: 186 raise ValueError("%s must be one of the following " 187 "options: %s" % (attr, 188 ', '.join(options))) 189 setattr(self, attr, val) 190 191 192 def _set_bool(self, attr, val): 193 val = str(val).lower() 194 if val == "false": 195 val = False 196 elif val == "true": 197 val = True 198 else: 199 msg = "%s must be either true or false" % attr 200 raise ValueError(msg) 201 setattr(self, attr, val) 202 203 204 def _set_int(self, attr, val, min=None, max=None): 205 val = int(val) 206 if min is not None and min > val: 207 raise ValueError("%s is %d, which is below the " 208 "minimum of %d" % (attr, val, min)) 209 if max is not None and max < val: 210 raise ValueError("%s is %d, which is above the " 211 "maximum of %d" % (attr, val, max)) 212 setattr(self, attr, val) 213 214 215 def _set_set(self, attr, val): 216 val = str(val) 217 items = [x.strip() for x in val.split(',') if x.strip()] 218 setattr(self, attr, set(items)) 219 220 221 def set_author(self, val): 222 self._set_string('author', val) 223 224 225 def set_dependencies(self, val): 226 self._set_set('dependencies', val) 227 228 229 def set_doc(self, val): 230 self._set_string('doc', val) 231 232 233 def set_name(self, val): 234 self._set_string('name', val) 235 236 237 def set_run_verify(self, val): 238 self._set_bool('run_verify', val) 239 240 241 def set_sync_count(self, val): 242 self._set_int('sync_count', val, min=1) 243 244 245 def set_suite(self, val): 246 self._set_string('suite', val) 247 248 249 def set_time(self, val): 250 self._set_option('time', val, ControlData.TEST_TIME_LIST) 251 252 253 def set_test_class(self, val): 254 self._set_string('test_class', val.lower()) 255 256 257 def set_test_category(self, val): 258 self._set_string('test_category', val.lower()) 259 260 261 def set_test_type(self, val): 262 self._set_option('test_type', val, list(CONTROL_TYPE.names)) 263 264 265 def set_test_parameters(self, val): 266 self._set_set('test_parameters', val) 267 268 269 def set_job_retries(self, val): 270 self._set_int('job_retries', val) 271 272 273 def set_bug_template(self, val): 274 if type(val) == dict: 275 setattr(self, 'bug_template', val) 276 277 278 def set_require_ssp(self, val): 279 self._set_bool('require_ssp', val) 280 281 282 def set_build(self, val): 283 self._set_string('build', val) 284 285 286 def set_builds(self, val): 287 if type(val) == dict: 288 setattr(self, 'builds', val) 289 290 def set_max_result_size_kb(self, val): 291 self._set_int('max_result_size_KB', val) 292 293 def set_priority(self, val): 294 self._set_int('priority', val) 295 296 def set_fast(self, val): 297 self._set_bool('fast', val) 298 299 def set_update_type(self, val): 300 self._set_string('update_type', val) 301 302 def set_source_release(self, val): 303 self._set_string('source_release', val) 304 305 def set_target_release(self, val): 306 self._set_string('target_release', val) 307 308 def set_target_payload_uri(self, val): 309 self._set_string('target_payload_uri', val) 310 311 def set_source_payload_uri(self, val): 312 self._set_string('source_payload_uri', val) 313 314 def set_source_archive_uri(self, val): 315 self._set_string('source_archive_uri', val) 316 317 def set_attributes(self, val): 318 self._set_set('attributes', val) 319 320 321def _extract_const(expr): 322 assert (expr.__class__ == ast.Str) 323 if six.PY2: 324 assert (expr.s.__class__ in (str, int, float, unicode)) 325 else: 326 assert (expr.s.__class__ in (str, int, float)) 327 return str(expr.s).strip() 328 329 330def _extract_dict(expr): 331 assert (expr.__class__ == ast.Dict) 332 assert (expr.keys.__class__ == list) 333 cf_dict = {} 334 for key, value in zip(expr.keys, expr.values): 335 try: 336 key = _extract_const(key) 337 val = _extract_expression(value) 338 except (AssertionError, ValueError): 339 pass 340 else: 341 cf_dict[key] = val 342 return cf_dict 343 344 345def _extract_list(expr): 346 assert (expr.__class__ == ast.List) 347 list_values = [] 348 for value in expr.elts: 349 try: 350 list_values.append(_extract_expression(value)) 351 except (AssertionError, ValueError): 352 pass 353 return list_values 354 355 356def _extract_name(expr): 357 assert (expr.__class__ == ast.Name) 358 assert (expr.id in ('False', 'True', 'None')) 359 return str(expr.id) 360 361 362def _extract_expression(expr): 363 if expr.__class__ == ast.Str: 364 return _extract_const(expr) 365 if expr.__class__ == ast.Name: 366 return _extract_name(expr) 367 if expr.__class__ == ast.Dict: 368 return _extract_dict(expr) 369 if expr.__class__ == ast.List: 370 return _extract_list(expr) 371 if expr.__class__ == ast.Num: 372 return expr.n 373 if six.PY3 and expr.__class__ == ast.NameConstant: 374 return expr.value 375 if six.PY3 and expr.__class__ == ast.Constant: 376 try: 377 return expr.value.strip() 378 except Exception: 379 return expr.value 380 raise ValueError('Unknown rval %s' % expr) 381 382 383def _extract_assignment(n): 384 assert (n.__class__ == ast.Assign) 385 assert (len(n.targets) == 1) 386 assert (n.targets[0].__class__ == ast.Name) 387 val = _extract_expression(n.value) 388 key = n.targets[0].id.lower() 389 return (key, val) 390 391 392def parse_control_string(control, raise_warnings=False, path=''): 393 """Parse a control file from a string. 394 395 @param control: string containing the text of a control file. 396 @param raise_warnings: True iff ControlData should raise an error on 397 warnings about control file contents. 398 @param path: string path to the control file. 399 400 """ 401 try: 402 mod = ast.parse(control) 403 except SyntaxError as e: 404 logging.error('Syntax error (%s) while parsing control string:', e) 405 lines = control.split('\n') 406 for n, l in enumerate(lines): 407 logging.error('Line %d: %s', n + 1, l) 408 raise ControlVariableException("Error parsing data because %s" % e) 409 return finish_parse(mod, path, raise_warnings) 410 411 412def parse_control(path, raise_warnings=False): 413 try: 414 with open(path, 'r') as r: 415 mod = ast.parse(r.read()) 416 except SyntaxError as e: 417 raise ControlVariableException("Error parsing %s because %s" % 418 (path, e)) 419 return finish_parse(mod, path, raise_warnings) 420 421 422def _try_extract_assignment(node, variables): 423 """Try to extract assignment from the given node. 424 425 @param node: An Assign object. 426 @param variables: Dictionary to store the parsed assignments. 427 """ 428 try: 429 key, val = _extract_assignment(node) 430 variables[key] = val 431 except (AssertionError, ValueError) as e: 432 pass 433 434 435def finish_parse(mod, path, raise_warnings): 436 assert (mod.__class__ == ast.Module) 437 assert (mod.body.__class__ == list) 438 439 variables = {} 440 injection_variables = {} 441 for n in mod.body: 442 if (n.__class__ == ast.FunctionDef and re.match('step\d+', n.name)): 443 vars_in_step = {} 444 for sub_node in n.body: 445 _try_extract_assignment(sub_node, vars_in_step) 446 if vars_in_step: 447 # Empty the vars collection so assignments from multiple steps 448 # won't be mixed. 449 variables.clear() 450 variables.update(vars_in_step) 451 else: 452 _try_extract_assignment(n, injection_variables) 453 454 variables.update(injection_variables) 455 return ControlData(variables, path, raise_warnings) 456