1# pylint: disable-msg=C0111 2 3# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7__author__ = 'cmasone@chromium.org (Chris Masone)' 8 9import common 10import ConfigParser 11import datetime 12import logging 13import os 14import shutil 15 16from autotest_lib.frontend.afe import models 17from autotest_lib.client.common_lib import control_data 18from autotest_lib.client.common_lib import error 19from autotest_lib.client.common_lib import global_config 20from autotest_lib.client.common_lib import priorities 21from autotest_lib.client.common_lib import time_utils 22from autotest_lib.client.common_lib.cros import dev_server 23from autotest_lib.client.common_lib.cros.graphite import autotest_stats 24from autotest_lib.frontend.afe import rpc_utils 25from autotest_lib.server import utils 26from autotest_lib.server.cros import provision 27from autotest_lib.server.cros.dynamic_suite import constants 28from autotest_lib.server.cros.dynamic_suite import control_file_getter 29from autotest_lib.server.cros.dynamic_suite import tools 30from autotest_lib.server.cros.dynamic_suite.suite import Suite 31from autotest_lib.server.hosts import moblab_host 32from autotest_lib.site_utils import host_history 33from autotest_lib.site_utils import job_history 34from autotest_lib.site_utils import server_manager_utils 35from autotest_lib.site_utils import stable_version_utils 36 37 38_CONFIG = global_config.global_config 39MOBLAB_BOTO_LOCATION = '/home/moblab/.boto' 40 41# Relevant CrosDynamicSuiteExceptions are defined in client/common_lib/error.py. 42 43 44def canonicalize_suite_name(suite_name): 45 # Do not change this naming convention without updating 46 # site_utils.parse_job_name. 47 return 'test_suites/control.%s' % suite_name 48 49 50def formatted_now(): 51 return datetime.datetime.now().strftime(time_utils.TIME_FMT) 52 53 54def _get_control_file_contents_by_name(build, ds, suite_name): 55 """Return control file contents for |suite_name|. 56 57 Query the dev server at |ds| for the control file |suite_name|, included 58 in |build| for |board|. 59 60 @param build: unique name by which to refer to the image from now on. 61 @param ds: a dev_server.DevServer instance to fetch control file with. 62 @param suite_name: canonicalized suite name, e.g. test_suites/control.bvt. 63 @raises ControlFileNotFound if a unique suite control file doesn't exist. 64 @raises NoControlFileList if we can't list the control files at all. 65 @raises ControlFileEmpty if the control file exists on the server, but 66 can't be read. 67 68 @return the contents of the desired control file. 69 """ 70 getter = control_file_getter.DevServerGetter.create(build, ds) 71 timer = autotest_stats.Timer('control_files.parse.%s.%s' % 72 (ds.get_server_name(ds.url() 73 ).replace('.', '_'), 74 suite_name.rsplit('.')[-1])) 75 # Get the control file for the suite. 76 try: 77 with timer: 78 control_file_in = getter.get_control_file_contents_by_name( 79 suite_name) 80 except error.CrosDynamicSuiteException as e: 81 raise type(e)("%s while testing %s." % (e, build)) 82 if not control_file_in: 83 raise error.ControlFileEmpty( 84 "Fetching %s returned no data." % suite_name) 85 # Force control files to only contain ascii characters. 86 try: 87 control_file_in.encode('ascii') 88 except UnicodeDecodeError as e: 89 raise error.ControlFileMalformed(str(e)) 90 91 return control_file_in 92 93 94def _stage_build_artifacts(build): 95 """ 96 Ensure components of |build| necessary for installing images are staged. 97 98 @param build image we want to stage. 99 100 @raises StageControlFileFailure: if the dev server throws 500 while staging 101 suite control files. 102 103 @return: dev_server.ImageServer instance to use with this build. 104 @return: timings dictionary containing staging start/end times. 105 """ 106 timings = {} 107 # Ensure components of |build| necessary for installing images are staged 108 # on the dev server. However set synchronous to False to allow other 109 # components to be downloaded in the background. 110 ds = dev_server.ImageServer.resolve(build) 111 timings[constants.DOWNLOAD_STARTED_TIME] = formatted_now() 112 timer = autotest_stats.Timer('control_files.stage.%s' % ( 113 ds.get_server_name(ds.url()).replace('.', '_'))) 114 try: 115 with timer: 116 ds.stage_artifacts(build, ['test_suites']) 117 except dev_server.DevServerException as e: 118 raise error.StageControlFileFailure( 119 "Failed to stage %s: %s" % (build, e)) 120 timings[constants.PAYLOAD_FINISHED_TIME] = formatted_now() 121 return (ds, timings) 122 123 124@rpc_utils.route_rpc_to_master 125def create_suite_job(name='', board='', build='', pool='', control_file='', 126 check_hosts=True, num=None, file_bugs=False, timeout=24, 127 timeout_mins=None, priority=priorities.Priority.DEFAULT, 128 suite_args=None, wait_for_results=True, job_retry=False, 129 max_retries=None, max_runtime_mins=None, suite_min_duts=0, 130 offload_failures_only=False, builds={}, 131 test_source_build=None, run_prod_code=False, **kwargs): 132 """ 133 Create a job to run a test suite on the given device with the given image. 134 135 When the timeout specified in the control file is reached, the 136 job is guaranteed to have completed and results will be available. 137 138 @param name: The test name if control_file is supplied, otherwise the name 139 of the test suite to run, e.g. 'bvt'. 140 @param board: the kind of device to run the tests on. 141 @param build: unique name by which to refer to the image from now on. 142 @param builds: the builds to install e.g. 143 {'cros-version:': 'x86-alex-release/R18-1655.0.0', 144 'fw-version:': 'x86-alex-firmware/R36-5771.50.0', 145 'fwro-version:': 'x86-alex-firmware/R36-5771.49.0'} 146 If builds is given a value, it overrides argument build. 147 @param test_source_build: Build that contains the server-side test code. 148 @param pool: Specify the pool of machines to use for scheduling 149 purposes. 150 @param check_hosts: require appropriate live hosts to exist in the lab. 151 @param num: Specify the number of machines to schedule across (integer). 152 Leave unspecified or use None to use default sharding factor. 153 @param file_bugs: File a bug on each test failure in this suite. 154 @param timeout: The max lifetime of this suite, in hours. 155 @param timeout_mins: The max lifetime of this suite, in minutes. Takes 156 priority over timeout. 157 @param priority: Integer denoting priority. Higher is more important. 158 @param suite_args: Optional arguments which will be parsed by the suite 159 control file. Used by control.test_that_wrapper to 160 determine which tests to run. 161 @param wait_for_results: Set to False to run the suite job without waiting 162 for test jobs to finish. Default is True. 163 @param job_retry: Set to True to enable job-level retry. Default is False. 164 @param max_retries: Integer, maximum job retries allowed at suite level. 165 None for no max. 166 @param max_runtime_mins: Maximum amount of time a job can be running in 167 minutes. 168 @param suite_min_duts: Integer. Scheduler will prioritize getting the 169 minimum number of machines for the suite when it is 170 competing with another suite that has a higher 171 priority but already got minimum machines it needs. 172 @param offload_failures_only: Only enable gs_offloading for failed jobs. 173 @param run_prod_code: If True, the suite will run the test code that 174 lives in prod aka the test code currently on the 175 lab servers. If False, the control files and test 176 code for this suite run will be retrieved from the 177 build artifacts. 178 @param kwargs: extra keyword args. NOT USED. 179 180 @raises ControlFileNotFound: if a unique suite control file doesn't exist. 181 @raises NoControlFileList: if we can't list the control files at all. 182 @raises StageControlFileFailure: If the dev server throws 500 while 183 staging test_suites. 184 @raises ControlFileEmpty: if the control file exists on the server, but 185 can't be read. 186 187 @return: the job ID of the suite; -1 on error. 188 """ 189 if type(num) is not int and num is not None: 190 raise error.SuiteArgumentException('Ill specified num argument %r. ' 191 'Must be an integer or None.' % num) 192 if num == 0: 193 logging.warning("Can't run on 0 hosts; using default.") 194 num = None 195 196 # TODO(dshi): crbug.com/496782 Remove argument build and its reference after 197 # R45 falls out of stable channel. 198 if build and not builds: 199 builds = {provision.CROS_VERSION_PREFIX: build} 200 # TODO(dshi): crbug.com/497236 Remove this check after firmware ro provision 201 # is supported in Autotest. 202 if provision.FW_RO_VERSION_PREFIX in builds: 203 raise error.SuiteArgumentException( 204 'Updating RO firmware is not supported yet.') 205 # Default test source build to CrOS build if it's not specified. 206 test_source_build = Suite.get_test_source_build( 207 builds, test_source_build=test_source_build) 208 209 suite_name = canonicalize_suite_name(name) 210 if run_prod_code: 211 ds = dev_server.ImageServer.resolve(build) 212 keyvals = {} 213 getter = control_file_getter.FileSystemGetter( 214 [_CONFIG.get_config_value('SCHEDULER', 215 'drone_installation_directory')]) 216 control_file = getter.get_control_file_contents_by_name(suite_name) 217 else: 218 (ds, keyvals) = _stage_build_artifacts(test_source_build) 219 keyvals[constants.SUITE_MIN_DUTS_KEY] = suite_min_duts 220 221 if not control_file: 222 # No control file was supplied so look it up from the build artifacts. 223 suite_name = canonicalize_suite_name(name) 224 control_file = _get_control_file_contents_by_name(test_source_build, 225 ds, suite_name) 226 # Do not change this naming convention without updating 227 # site_utils.parse_job_name. 228 name = '%s-%s' % (test_source_build, suite_name) 229 230 timeout_mins = timeout_mins or timeout * 60 231 max_runtime_mins = max_runtime_mins or timeout * 60 232 233 if not board: 234 board = utils.ParseBuildName(builds[provision.CROS_VERSION_PREFIX])[0] 235 236 # TODO(dshi): crbug.com/496782 Remove argument build and its reference after 237 # R45 falls out of stable channel. 238 # Prepend build and board to the control file. 239 inject_dict = {'board': board, 240 'build': builds.get(provision.CROS_VERSION_PREFIX), 241 'builds': builds, 242 'check_hosts': check_hosts, 243 'pool': pool, 244 'num': num, 245 'file_bugs': file_bugs, 246 'timeout': timeout, 247 'timeout_mins': timeout_mins, 248 'devserver_url': ds.url(), 249 'priority': priority, 250 'suite_args' : suite_args, 251 'wait_for_results': wait_for_results, 252 'job_retry': job_retry, 253 'max_retries': max_retries, 254 'max_runtime_mins': max_runtime_mins, 255 'offload_failures_only': offload_failures_only, 256 'test_source_build': test_source_build, 257 'run_prod_code': run_prod_code 258 } 259 260 control_file = tools.inject_vars(inject_dict, control_file) 261 262 return rpc_utils.create_job_common(name, 263 priority=priority, 264 timeout_mins=timeout_mins, 265 max_runtime_mins=max_runtime_mins, 266 control_type='Server', 267 control_file=control_file, 268 hostless=True, 269 keyvals=keyvals) 270 271 272# TODO: hide the following rpcs under is_moblab 273def moblab_only(func): 274 """Ensure moblab specific functions only run on Moblab devices.""" 275 def verify(*args, **kwargs): 276 if not utils.is_moblab(): 277 raise error.RPCException('RPC: %s can only run on Moblab Systems!', 278 func.__name__) 279 return func(*args, **kwargs) 280 return verify 281 282 283@moblab_only 284def get_config_values(): 285 """Returns all config values parsed from global and shadow configs. 286 287 Config values are grouped by sections, and each section is composed of 288 a list of name value pairs. 289 """ 290 sections =_CONFIG.get_sections() 291 config_values = {} 292 for section in sections: 293 config_values[section] = _CONFIG.config.items(section) 294 return rpc_utils.prepare_for_serialization(config_values) 295 296 297@moblab_only 298def update_config_handler(config_values): 299 """ 300 Update config values and override shadow config. 301 302 @param config_values: See get_moblab_settings(). 303 """ 304 original_config = global_config.global_config_class() 305 original_config.set_config_files(shadow_file='') 306 new_shadow = ConfigParser.RawConfigParser() 307 for section, config_value_list in config_values.iteritems(): 308 for key, value in config_value_list: 309 if original_config.get_config_value(section, key, 310 default='', 311 allow_blank=True) != value: 312 if not new_shadow.has_section(section): 313 new_shadow.add_section(section) 314 new_shadow.set(section, key, value) 315 if not _CONFIG.shadow_file or not os.path.exists(_CONFIG.shadow_file): 316 raise error.RPCException('Shadow config file does not exist.') 317 318 with open(_CONFIG.shadow_file, 'w') as config_file: 319 new_shadow.write(config_file) 320 # TODO (sbasi) crbug.com/403916 - Remove the reboot command and 321 # instead restart the services that rely on the config values. 322 os.system('sudo reboot') 323 324 325@moblab_only 326def reset_config_settings(): 327 with open(_CONFIG.shadow_file, 'w') as config_file: 328 pass 329 os.system('sudo reboot') 330 331 332@moblab_only 333def set_boto_key(boto_key): 334 """Update the boto_key file. 335 336 @param boto_key: File name of boto_key uploaded through handle_file_upload. 337 """ 338 if not os.path.exists(boto_key): 339 raise error.RPCException('Boto key: %s does not exist!' % boto_key) 340 shutil.copyfile(boto_key, moblab_host.MOBLAB_BOTO_LOCATION) 341 342 343@moblab_only 344def set_launch_control_key(launch_control_key): 345 """Update the launch_control_key file. 346 347 @param launch_control_key: File name of launch_control_key uploaded through 348 handle_file_upload. 349 """ 350 if not os.path.exists(launch_control_key): 351 raise error.RPCException('Launch Control key: %s does not exist!' % 352 launch_control_key) 353 shutil.copyfile(launch_control_key, 354 moblab_host.MOBLAB_LAUNCH_CONTROL_KEY_LOCATION) 355 # Restart the devserver service. 356 os.system('sudo restart moblab-devserver-init') 357 358 359def get_job_history(**filter_data): 360 """Get history of the job, including the special tasks executed for the job 361 362 @param filter_data: filter for the call, should at least include 363 {'job_id': [job id]} 364 @returns: JSON string of the job's history, including the information such 365 as the hosts run the job and the special tasks executed before 366 and after the job. 367 """ 368 job_id = filter_data['job_id'] 369 job_info = job_history.get_job_info(job_id) 370 return rpc_utils.prepare_for_serialization(job_info.get_history()) 371 372 373def get_host_history(start_time, end_time, hosts=None, board=None, pool=None): 374 """Get history of a list of host. 375 376 The return is a JSON string of host history for each host, for example, 377 {'172.22.33.51': [{'status': 'Resetting' 378 'start_time': '2014-08-07 10:02:16', 379 'end_time': '2014-08-07 10:03:16', 380 'log_url': 'http://autotest/reset-546546/debug', 381 'dbg_str': 'Task: Special Task 19441991 (host ...)'}, 382 {'status': 'Running' 383 'start_time': '2014-08-07 10:03:18', 384 'end_time': '2014-08-07 10:13:00', 385 'log_url': 'http://autotest/reset-546546/debug', 386 'dbg_str': 'HQE: 15305005, for job: 14995562'} 387 ] 388 } 389 @param start_time: start time to search for history, can be string value or 390 epoch time. 391 @param end_time: end time to search for history, can be string value or 392 epoch time. 393 @param hosts: A list of hosts to search for history. Default is None. 394 @param board: board type of hosts. Default is None. 395 @param pool: pool type of hosts. Default is None. 396 @returns: JSON string of the host history. 397 """ 398 return rpc_utils.prepare_for_serialization( 399 host_history.get_history_details( 400 start_time=start_time, end_time=end_time, 401 hosts=hosts, board=board, pool=pool, 402 process_pool_size=4)) 403 404 405def shard_heartbeat(shard_hostname, jobs=(), hqes=(), known_job_ids=(), 406 known_host_ids=(), known_host_statuses=()): 407 """Receive updates for job statuses from shards and assign hosts and jobs. 408 409 @param shard_hostname: Hostname of the calling shard 410 @param jobs: Jobs in serialized form that should be updated with newer 411 status from a shard. 412 @param hqes: Hostqueueentries in serialized form that should be updated with 413 newer status from a shard. Note that for every hostqueueentry 414 the corresponding job must be in jobs. 415 @param known_job_ids: List of ids of jobs the shard already has. 416 @param known_host_ids: List of ids of hosts the shard already has. 417 @param known_host_statuses: List of statuses of hosts the shard already has. 418 419 @returns: Serialized representations of hosts, jobs, suite job keyvals 420 and their dependencies to be inserted into a shard's database. 421 """ 422 # The following alternatives to sending host and job ids in every heartbeat 423 # have been considered: 424 # 1. Sending the highest known job and host ids. This would work for jobs: 425 # Newer jobs always have larger ids. Also, if a job is not assigned to a 426 # particular shard during a heartbeat, it never will be assigned to this 427 # shard later. 428 # This is not true for hosts though: A host that is leased won't be sent 429 # to the shard now, but might be sent in a future heartbeat. This means 430 # sometimes hosts should be transfered that have a lower id than the 431 # maximum host id the shard knows. 432 # 2. Send the number of jobs/hosts the shard knows to the master in each 433 # heartbeat. Compare these to the number of records that already have 434 # the shard_id set to this shard. In the normal case, they should match. 435 # In case they don't, resend all entities of that type. 436 # This would work well for hosts, because there aren't that many. 437 # Resending all jobs is quite a big overhead though. 438 # Also, this approach might run into edge cases when entities are 439 # ever deleted. 440 # 3. Mixtures of the above: Use 1 for jobs and 2 for hosts. 441 # Using two different approaches isn't consistent and might cause 442 # confusion. Also the issues with the case of deletions might still 443 # occur. 444 # 445 # The overhead of sending all job and host ids in every heartbeat is low: 446 # At peaks one board has about 1200 created but unfinished jobs. 447 # See the numbers here: http://goo.gl/gQCGWH 448 # Assuming that job id's have 6 digits and that json serialization takes a 449 # comma and a space as overhead, the traffic per id sent is about 8 bytes. 450 # If 5000 ids need to be sent, this means 40 kilobytes of traffic. 451 # A NOT IN query with 5000 ids took about 30ms in tests made. 452 # These numbers seem low enough to outweigh the disadvantages of the 453 # solutions described above. 454 timer = autotest_stats.Timer('shard_heartbeat') 455 with timer: 456 shard_obj = rpc_utils.retrieve_shard(shard_hostname=shard_hostname) 457 rpc_utils.persist_records_sent_from_shard(shard_obj, jobs, hqes) 458 assert len(known_host_ids) == len(known_host_statuses) 459 for i in range(len(known_host_ids)): 460 host_model = models.Host.objects.get(pk=known_host_ids[i]) 461 if host_model.status != known_host_statuses[i]: 462 host_model.status = known_host_statuses[i] 463 host_model.save() 464 465 hosts, jobs, suite_keyvals = rpc_utils.find_records_for_shard( 466 shard_obj, known_job_ids=known_job_ids, 467 known_host_ids=known_host_ids) 468 return { 469 'hosts': [host.serialize() for host in hosts], 470 'jobs': [job.serialize() for job in jobs], 471 'suite_keyvals': [kv.serialize() for kv in suite_keyvals], 472 } 473 474 475def get_shards(**filter_data): 476 """Return a list of all shards. 477 478 @returns A sequence of nested dictionaries of shard information. 479 """ 480 shards = models.Shard.query_objects(filter_data) 481 serialized_shards = rpc_utils.prepare_rows_as_nested_dicts(shards, ()) 482 for serialized, shard in zip(serialized_shards, shards): 483 serialized['labels'] = [label.name for label in shard.labels.all()] 484 485 return serialized_shards 486 487 488def add_shard(hostname, labels): 489 """Add a shard and start running jobs on it. 490 491 @param hostname: The hostname of the shard to be added; needs to be unique. 492 @param labels: Board labels separated by a comma. Jobs of one of the labels 493 will be assigned to the shard. 494 495 @raises error.RPCException: If label provided doesn't start with `board:` 496 @raises model_logic.ValidationError: If a shard with the given hostname 497 already exists. 498 @raises models.Label.DoesNotExist: If the label specified doesn't exist. 499 """ 500 labels = labels.split(',') 501 label_models = [] 502 for label in labels: 503 if not label.startswith('board:'): 504 raise error.RPCException('Sharding only supports for `board:.*` ' 505 'labels.') 506 # Fetch label first, so shard isn't created when label doesn't exist. 507 label_models.append(models.Label.smart_get(label)) 508 509 shard = models.Shard.add_object(hostname=hostname) 510 for label in label_models: 511 shard.labels.add(label) 512 return shard.id 513 514 515def delete_shard(hostname): 516 """Delete a shard and reclaim all resources from it. 517 518 This claims back all assigned hosts from the shard. To ensure all DUTs are 519 in a sane state, a Repair task is scheduled for them. This reboots the DUTs 520 and therefore clears all running processes that might be left. 521 522 The shard_id of jobs of that shard will be set to None. 523 524 The status of jobs that haven't been reported to be finished yet, will be 525 lost. The master scheduler will pick up the jobs and execute them. 526 527 @param hostname: Hostname of the shard to delete. 528 """ 529 shard = rpc_utils.retrieve_shard(shard_hostname=hostname) 530 531 # TODO(beeps): Power off shard 532 533 # For ChromeOS hosts, repair reboots the DUT. 534 # Repair will excalate through multiple repair steps and will verify the 535 # success after each of them. Anyway, it will always run at least the first 536 # one, which includes a reboot. 537 # After a reboot we can be sure no processes from prior tests that were run 538 # by a shard are still running on the DUT. 539 # Important: Don't just set the status to Repair Failed, as that would run 540 # Verify first, before doing any repair measures. Verify would probably 541 # succeed, so this wouldn't change anything on the DUT. 542 for host in models.Host.objects.filter(shard=shard): 543 models.SpecialTask.objects.create( 544 task=models.SpecialTask.Task.REPAIR, 545 host=host, 546 requested_by=models.User.current_user()) 547 models.Host.objects.filter(shard=shard).update(shard=None) 548 549 models.Job.objects.filter(shard=shard).update(shard=None) 550 551 shard.labels.clear() 552 553 shard.delete() 554 555 556def get_servers(hostname=None, role=None, status=None): 557 """Get a list of servers with matching role and status. 558 559 @param hostname: FQDN of the server. 560 @param role: Name of the server role, e.g., drone, scheduler. Default to 561 None to match any role. 562 @param status: Status of the server, e.g., primary, backup, repair_required. 563 Default to None to match any server status. 564 565 @raises error.RPCException: If server database is not used. 566 @return: A list of server names for servers with matching role and status. 567 """ 568 if not server_manager_utils.use_server_db(): 569 raise error.RPCException('Server database is not enabled. Please try ' 570 'retrieve servers from global config.') 571 servers = server_manager_utils.get_servers(hostname=hostname, role=role, 572 status=status) 573 return [s.get_details() for s in servers] 574 575 576@rpc_utils.route_rpc_to_master 577def get_stable_version(board=stable_version_utils.DEFAULT, android=False): 578 """Get stable version for the given board. 579 580 @param board: Name of the board. 581 @param android: If True, the given board is an Android-based device. If 582 False, assume its a Chrome OS-based device. 583 584 @return: Stable version of the given board. Return global configure value 585 of CROS.stable_cros_version if stable_versinos table does not have 586 entry of board DEFAULT. 587 """ 588 return stable_version_utils.get(board=board, android=android) 589 590 591@rpc_utils.route_rpc_to_master 592def get_all_stable_versions(): 593 """Get stable versions for all boards. 594 595 @return: A dictionary of board:version. 596 """ 597 return stable_version_utils.get_all() 598 599 600@rpc_utils.route_rpc_to_master 601def set_stable_version(version, board=stable_version_utils.DEFAULT): 602 """Modify stable version for the given board. 603 604 @param version: The new value of stable version for given board. 605 @param board: Name of the board, default to value `DEFAULT`. 606 """ 607 stable_version_utils.set(version=version, board=board) 608 609 610@rpc_utils.route_rpc_to_master 611def delete_stable_version(board): 612 """Modify stable version for the given board. 613 614 Delete a stable version entry in afe_stable_versions table for a given 615 board, so default stable version will be used. 616 617 @param board: Name of the board. 618 """ 619 stable_version_utils.delete(board=board) 620 621 622def get_tests_by_build(build): 623 """Get the tests that are available for the specified build. 624 625 @param build: unique name by which to refer to the image. 626 627 @return: A sorted list of all tests that are in the build specified. 628 """ 629 # Stage the test artifacts. 630 try: 631 ds = dev_server.ImageServer.resolve(build) 632 build = ds.translate(build) 633 except dev_server.DevServerException as e: 634 raise ValueError('Could not resolve build %s: %s' % (build, e)) 635 636 try: 637 ds.stage_artifacts(build, ['test_suites']) 638 except dev_server.DevServerException as e: 639 raise error.StageControlFileFailure( 640 'Failed to stage %s: %s' % (build, e)) 641 642 # Collect the control files specified in this build 643 cfile_getter = control_file_getter.DevServerGetter.create(build, ds) 644 control_file_list = cfile_getter.get_control_file_list() 645 646 test_objects = [] 647 _id = 0 648 for control_file_path in control_file_list: 649 # Read and parse the control file 650 control_file = cfile_getter.get_control_file_contents( 651 control_file_path) 652 control_obj = control_data.parse_control_string(control_file) 653 654 # Extract the values needed for the AFE from the control_obj. 655 # The keys list represents attributes in the control_obj that 656 # are required by the AFE 657 keys = ['author', 'doc', 'name', 'time', 'test_type', 'experimental', 658 'test_category', 'test_class', 'dependencies', 'run_verify', 659 'sync_count', 'job_retries', 'retries', 'path'] 660 661 test_object = {} 662 for key in keys: 663 test_object[key] = getattr(control_obj, key) if hasattr( 664 control_obj, key) else '' 665 666 # Unfortunately, the AFE expects different key-names for certain 667 # values, these must be corrected to avoid the risk of tests 668 # being omitted by the AFE. 669 # The 'id' is an additional value used in the AFE. 670 # The control_data parsing does not reference 'run_reset', but it 671 # is also used in the AFE and defaults to True. 672 test_object['id'] = _id 673 test_object['run_reset'] = True 674 test_object['description'] = test_object.get('doc', '') 675 test_object['test_time'] = test_object.get('time', 0) 676 test_object['test_retry'] = test_object.get('retries', 0) 677 678 # Fix the test name to be consistent with the current presentation 679 # of test names in the AFE. 680 testpath, subname = os.path.split(control_file_path) 681 testname = os.path.basename(testpath) 682 subname = subname.split('.')[1:] 683 if subname: 684 testname = '%s:%s' % (testname, ':'.join(subname)) 685 686 test_object['name'] = testname 687 688 # Correct the test path as parse_control_string sets an empty string. 689 test_object['path'] = control_file_path 690 691 _id += 1 692 test_objects.append(test_object) 693 694 test_objects = sorted(test_objects, key=lambda x: x.get('name')) 695 return rpc_utils.prepare_for_serialization(test_objects) 696