1# Copyright 2015 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Argument validation for the DUT deployment tool. 6 7Arguments for the DUT deployment commands require more processing than 8can readily be done by `ArgumentParser.parse_args()`. The post-parsing 9validation process not only checks that arguments have allowable values, 10but also may perform a dialog with the user to ask for missing arguments. 11Finally, it adds in information needed by `install.install_duts()`. 12 13The interactive dialog is invoked if the board and hostnames are omitted 14from the command line. The dialog, if invoked, will get the following 15information from the user: 16 * (required) Board of the DUTs to be deployed. 17 * (required) Hostnames of the DUTs to be deployed. 18 * (optional) Version of the test image to be made the stable 19 repair image for the board to be deployed. If omitted, the 20 existing setting is retained. 21""" 22 23import collections 24import csv 25import datetime 26import os 27import re 28import subprocess 29import sys 30 31import dateutil.tz 32 33import common 34from autotest_lib.server.hosts import servo_host 35 36# _BUILD_URI_FORMAT 37# A format template for a Google storage URI that designates 38# one build. The template is to be filled in with a board 39# name and build version number. 40 41_BUILD_URI_FORMAT = 'gs://chromeos-image-archive/%s-release/%s' 42 43 44# _BUILD_PATTERNS 45# For user convenience, argument parsing allows various formats 46# for build version strings. The function _normalize_build_name() 47# is used to convert the recognized syntaxes into the name as 48# it appears in Google storage. 49# 50# _BUILD_PATTERNS describe the recognized syntaxes for user-supplied 51# build versions, and information about how to convert them. See the 52# normalize function for details. 53# 54# For user-supplied build versions, the following forms are supported: 55# #### - Indicates a canary; equivalent to ####.0.0. 56# ####.#.# - A full build version without the leading R##- prefix. 57# R##-###.#.# - Canonical form of a build version. 58 59_BUILD_PATTERNS = [ 60 (re.compile(r'^R\d+-\d+\.\d+\.\d+$'), None), 61 (re.compile(r'^\d+\.\d+\.\d+$'), 'LATEST-%s'), 62 (re.compile(r'^\d+$'), 'LATEST-%s.0.0'), 63] 64 65 66# _VALID_HOSTNAME_PATTERNS 67# A list of REs describing patterns that are acceptable as names 68# for DUTs in the test lab. Names that don't match one of the 69# patterns will be rejected as invalid. 70 71_VALID_HOSTNAME_PATTERNS = [ 72 re.compile(r'chromeos\d+-row\d+-rack\d+-host\d+') 73] 74 75 76# _EXPECTED_NUMBER_OF_HOST_INFO 77# The number of items per line when parsing the hostname_file csv file. 78_EXPECTED_NUMBER_OF_HOST_INFO = 8 79 80# HostInfo 81# Namedtuple to store host info for processing when creating host in the afe. 82HostInfo = collections.namedtuple('HostInfo', ['hostname', 'host_attr_dict']) 83 84 85def _build_path_exists(board, buildpath): 86 """Return whether a given build file exists in Google storage. 87 88 The `buildpath` refers to a specific file associated with 89 release builds for `board`. The path may be one of the "LATEST" 90 files (e.g. "LATEST-7356.0.0"), or it could refer to a build 91 artifact (e.g. "R46-7356.0.0/image.zip"). 92 93 The function constructs the full GS URI from the arguments, and 94 then tests for its existence with `gsutil ls`. 95 96 @param board Board to be tested. 97 @param buildpath Partial path of a file in Google storage. 98 99 @return Return a true value iff the designated file exists. 100 """ 101 try: 102 gsutil_cmd = [ 103 'gsutil', 'ls', 104 _BUILD_URI_FORMAT % (board, buildpath) 105 ] 106 status = subprocess.call(gsutil_cmd, 107 stdout=open('/dev/null', 'w'), 108 stderr=subprocess.STDOUT) 109 return status == 0 110 except: 111 return False 112 113 114def _normalize_build_name(board, build): 115 """Convert a user-supplied build version to canonical form. 116 117 Canonical form looks like R##-####.#.#, e.g. R46-7356.0.0. 118 Acceptable user-supplied forms are describe under 119 _BUILD_PATTERNS, above. The returned value will be the name of 120 a directory containing build artifacts from a release builder 121 for the board. 122 123 Walk through `_BUILD_PATTERNS`, trying to convert a user 124 supplied build version name into a directory name for valid 125 build artifacts. Searching stops at the first pattern matched, 126 regardless of whether the designated build actually exists. 127 128 `_BUILD_PATTERNS` is a list of tuples. The first element of the 129 tuple is an RE describing a valid user input. The second 130 element of the tuple is a format pattern for a "LATEST" filename 131 in storage that can be used to obtain the full build version 132 associated with the user supplied version. If the second element 133 is `None`, the user supplied build version is already in canonical 134 form. 135 136 @param board Board to be tested. 137 @param build User supplied version name. 138 139 @return Return the name of a directory in canonical form, or 140 `None` if the build doesn't exist. 141 """ 142 for regex, fmt in _BUILD_PATTERNS: 143 if not regex.match(build): 144 continue 145 if fmt is not None: 146 try: 147 gsutil_cmd = [ 148 'gsutil', 'cat', 149 _BUILD_URI_FORMAT % (board, fmt % build) 150 ] 151 return subprocess.check_output( 152 gsutil_cmd, stderr=open('/dev/null', 'w')) 153 except: 154 return None 155 elif _build_path_exists(board, '%s/image.zip' % build): 156 return build 157 else: 158 return None 159 return None 160 161 162def _validate_board(board): 163 """Return whether a given board exists in Google storage. 164 165 For purposes of this function, a board exists if it has a 166 "LATEST-master" file in its release builder's directory. 167 168 N.B. For convenience, this function prints an error message 169 on stderr in certain failure cases. This is currently useful 170 for argument processing, but isn't really ideal if the callers 171 were to get more complicated. 172 173 @param board The board to be tested for existence. 174 @return Return a true value iff the board exists. 175 """ 176 # In this case, the board doesn't exist, but we don't want 177 # an error message. 178 if board is None: 179 return False 180 # Check Google storage; report failures on stderr. 181 if _build_path_exists(board, 'LATEST-master'): 182 return True 183 else: 184 sys.stderr.write('Board %s doesn\'t exist.\n' % board) 185 return False 186 187 188def _validate_build(board, build): 189 """Return whether a given build exists in Google storage. 190 191 N.B. For convenience, this function prints an error message 192 on stderr in certain failure cases. This is currently useful 193 for argument processing, but isn't really ideal if the callers 194 were to get more complicated. 195 196 @param board The board to be tested for a build 197 @param build The version of the build to be tested for. This 198 build may be in a user-specified (non-canonical) 199 form. 200 @return If the given board+build exists, return its canonical 201 (normalized) version string. If the build doesn't 202 exist, return a false value. 203 """ 204 canonical_build = _normalize_build_name(board, build) 205 if not canonical_build: 206 sys.stderr.write( 207 'Build %s is not a valid build version for %s.\n' % 208 (build, board)) 209 return canonical_build 210 211 212def _validate_hostname(hostname): 213 """Return whether a given hostname is valid for the test lab. 214 215 This is a sanity check meant to guarantee that host names follow 216 naming requirements for the test lab. 217 218 N.B. For convenience, this function prints an error message 219 on stderr in certain failure cases. This is currently useful 220 for argument processing, but isn't really ideal if the callers 221 were to get more complicated. 222 223 @param hostname The host name to be checked. 224 @return Return a true value iff the hostname is valid. 225 """ 226 for p in _VALID_HOSTNAME_PATTERNS: 227 if p.match(hostname): 228 return True 229 sys.stderr.write( 230 'Hostname %s doesn\'t match a valid location name.\n' % 231 hostname) 232 return False 233 234 235def _is_hostname_file_valid(hostname_file): 236 """Check that the hostname file is valid. 237 238 The hostname file is deemed valid if: 239 - the file exists. 240 - the file is non-empty. 241 242 @param hostname_file Filename of the hostname file to check. 243 244 @return `True` if the hostname file is valid, False otherse. 245 """ 246 return os.path.exists(hostname_file) and os.path.getsize(hostname_file) > 0 247 248 249def _validate_arguments(arguments): 250 """Check command line arguments, and account for defaults. 251 252 Check that all command-line argument constraints are satisfied. 253 If errors are found, they are reported on `sys.stderr`. 254 255 If there are any fields with defined defaults that couldn't be 256 calculated when we constructed the argument parser, calculate 257 them now. 258 259 @param arguments Parsed results from 260 `ArgumentParser.parse_args()`. 261 @return Return `True` if there are no errors to report, or 262 `False` if there are. 263 """ 264 # If both hostnames and hostname_file are specified, complain about that. 265 if arguments.hostnames and arguments.hostname_file: 266 sys.stderr.write( 267 'DUT hostnames and hostname file both specified, only ' 268 'specify one or the other.\n') 269 return False 270 if (arguments.hostname_file and 271 not _is_hostname_file_valid(arguments.hostname_file)): 272 sys.stderr.write( 273 'Specified hostname file must exist and be non-empty.\n') 274 return False 275 if (not arguments.hostnames and not arguments.hostname_file and 276 (arguments.board or arguments.build)): 277 sys.stderr.write( 278 'DUT hostnames are required with board or build.\n') 279 return False 280 if arguments.board is not None: 281 if not _validate_board(arguments.board): 282 return False 283 if (arguments.build is not None and 284 not _validate_build(arguments.board, arguments.build)): 285 return False 286 return True 287 288 289def _read_with_prompt(input, prompt): 290 """Print a prompt and then read a line of text. 291 292 @param input File-like object from which to read the line. 293 @param prompt String to print to stderr prior to reading. 294 @return Returns a string, stripped of whitespace. 295 """ 296 full_prompt = '%s> ' % prompt 297 sys.stderr.write(full_prompt) 298 return input.readline().strip() 299 300 301def _read_board(input, default_board): 302 """Read a valid board name from user input. 303 304 Prompt the user to supply a board name, and read one line. If 305 the line names a valid board, return the board name. If the 306 line is blank and `default_board` is a non-empty string, returns 307 `default_board`. Retry until a valid input is obtained. 308 309 `default_board` isn't checked; the caller is responsible for 310 ensuring its validity. 311 312 @param input File-like object from which to read the 313 board. 314 @param default_board Value to return if the user enters a 315 blank line. 316 @return Returns `default_board` or a validated board name. 317 """ 318 if default_board: 319 board_prompt = 'board name [%s]' % default_board 320 else: 321 board_prompt = 'board name' 322 new_board = None 323 while not _validate_board(new_board): 324 new_board = _read_with_prompt(input, board_prompt).lower() 325 if new_board: 326 sys.stderr.write('Checking for valid board.\n') 327 elif default_board: 328 return default_board 329 return new_board 330 331 332def _read_build(input, board): 333 """Read a valid build version from user input. 334 335 Prompt the user to supply a build version, and read one line. 336 If the line names an existing version for the given board, 337 return the canonical build version. If the line is blank, 338 return `None` (indicating the build shouldn't change). 339 340 @param input File-like object from which to read the build. 341 @param board Board for the build. 342 @return Returns canonical build version, or `None`. 343 """ 344 build = False 345 prompt = 'build version (optional)' 346 while not build: 347 build = _read_with_prompt(input, prompt) 348 if not build: 349 return None 350 sys.stderr.write('Checking for valid build.\n') 351 build = _validate_build(board, build) 352 return build 353 354 355def _read_model(input, default_model): 356 """Read a valid model name from user input. 357 358 Prompt the user to supply a model name, and read one line. If 359 the line names a valid model, return the model name. If the 360 line is blank and `default_model` is a non-empty string, returns 361 `default_model`. Retry until a valid input is obtained. 362 363 `default_model` isn't checked; the caller is responsible for 364 ensuring its validity. 365 366 @param input File-like object from which to read the 367 model. 368 @param default_model Value to return if the user enters a 369 blank line. 370 @return Returns `default_model` or a model name. 371 """ 372 model_prompt = 'model name' 373 if default_model: 374 model_prompt += ' [%s]' % default_model 375 new_model = None 376 # TODO(guocb): create a real model validator 377 _validate_model = lambda x: x 378 379 while not _validate_model(new_model): 380 new_model = _read_with_prompt(input, model_prompt).lower() 381 if new_model: 382 sys.stderr.write("It's your responsiblity to ensure validity of " 383 "model name.\n") 384 elif default_model: 385 return default_model 386 return new_model 387 388 389def _read_hostnames(input): 390 """Read a list of host names from user input. 391 392 Prompt the user to supply a list of host names. Any number of 393 lines are allowed; input is terminated at the first blank line. 394 Any number of hosts names are allowed on one line. Names are 395 separated by whitespace. 396 397 Only valid host names are accepted. Invalid host names are 398 ignored, and a warning is printed. 399 400 @param input File-like object from which to read the names. 401 @return Returns a list of validated host names. 402 """ 403 hostnames = [] 404 y_n = 'yes' 405 while not 'no'.startswith(y_n): 406 sys.stderr.write('enter hosts (blank line to end):\n') 407 while True: 408 new_hosts = input.readline().strip().split() 409 if not new_hosts: 410 break 411 for h in new_hosts: 412 if _validate_hostname(h): 413 hostnames.append(h) 414 if not hostnames: 415 sys.stderr.write('Must provide at least one hostname.\n') 416 continue 417 prompt = 'More hosts? [y/N]' 418 y_n = _read_with_prompt(input, prompt).lower() or 'no' 419 return hostnames 420 421 422def _read_arguments(input, arguments): 423 """Dialog to read all needed arguments from the user. 424 425 The user is prompted in turn for a board, a build, a model, and 426 hostnames. Responses are stored in `arguments`. The user is 427 given opportunity to accept or reject the responses before 428 continuing. 429 430 @param input File-like object from which to read user 431 responses. 432 @param arguments Namespace object returned from 433 `ArgumentParser.parse_args()`. Results are 434 stored here. 435 """ 436 y_n = 'no' 437 while not 'yes'.startswith(y_n): 438 arguments.board = _read_board(input, arguments.board) 439 arguments.build = _read_build(input, arguments.board) 440 arguments.model = _read_model(input, arguments.model) 441 prompt = '%s build %s? [Y/n]' % ( 442 arguments.board, arguments.build) 443 y_n = _read_with_prompt(input, prompt).lower() or 'yes' 444 arguments.hostnames = _read_hostnames(input) 445 446 447def _parse_hostname_file_line(hostname_file_row): 448 """ 449 Parse a line from the hostname_file and return a dict of the info. 450 451 @param hostname_file_row: List of strings from each line in the hostname 452 file. 453 454 @returns a NamedTuple of (hostname, host_attr_dict). host_attr_dict is a 455 dict of host attributes for the host. 456 """ 457 if len(hostname_file_row) != _EXPECTED_NUMBER_OF_HOST_INFO: 458 raise Exception('hostname_file line has unexpected number of items ' 459 '%d (expect %d): %s' % 460 (len(hostname_file_row), _EXPECTED_NUMBER_OF_HOST_INFO, 461 hostname_file_row)) 462 # The file will have the info in the following order: 463 # 0: board 464 # 1: dut hostname 465 # 2: dut/v4 mac address 466 # 3: dut ip 467 # 4: labstation hostname 468 # 5: servo serial 469 # 6: servo mac address 470 # 7: servo ip 471 return HostInfo( 472 hostname=hostname_file_row[1], 473 host_attr_dict={servo_host.SERVO_HOST_ATTR: hostname_file_row[4], 474 servo_host.SERVO_SERIAL_ATTR: hostname_file_row[5]}) 475 476 477def _get_upload_basename(arguments): 478 """Get base name for logs upload. 479 480 @param arguments Namespace object returned from argument parsing. 481 @return A filename as a string. 482 """ 483 timestamp = datetime.datetime.now(dateutil.tz.tzlocal()).isoformat() 484 return '{time}-{board}'.format(time=timestamp, board=arguments.board) 485 486 487def _parse_hostname_file(hostname_file): 488 """ 489 Parse the hostname_file and return a list of dicts for each line. 490 491 @param hostname_file: CSV file that contains all the goodies. 492 493 @returns a list of dicts where each line is broken down into a dict. 494 """ 495 host_info_list = [] 496 # First line will be the header, no need to parse that. 497 first_line_skipped = False 498 with open(hostname_file) as f: 499 hostname_file_reader = csv.reader(f) 500 for row in hostname_file_reader: 501 if not first_line_skipped: 502 first_line_skipped = True 503 continue 504 host_info_list.append(_parse_hostname_file_line(row)) 505 506 return host_info_list 507 508 509def validate_arguments(arguments): 510 """Validate parsed arguments for a repair or deployment command. 511 512 The `arguments` parameter represents a `Namespace` object returned 513 by `cmdparse.parse_command()`. Check this for mandatory arguments; 514 if they're missing, execute a dialog with the user to read them from 515 `sys.stdin`. 516 517 Once all arguments are known to be filled in, validate the values, 518 and fill in additional information that couldn't be processed at 519 parsing time. 520 521 @param arguments Standard `Namespace` object as returned by 522 `cmdparse.parse_command()`. 523 """ 524 if not arguments.board or not arguments.model: 525 _read_arguments(sys.stdin, arguments) 526 elif not _validate_arguments(arguments): 527 return None 528 529 arguments.upload_basename = _get_upload_basename(arguments) 530 if not arguments.logdir: 531 arguments.logdir = os.path.join(os.environ['HOME'], 532 'Documents', 533 arguments.upload_basename) 534 os.makedirs(arguments.logdir) 535 elif not os.path.isdir(arguments.logdir): 536 os.mkdir(arguments.logdir) 537 538 if arguments.hostname_file: 539 # Populate arguments.hostnames with the hostnames from the file. 540 hostname_file_info_list = _parse_hostname_file(arguments.hostname_file) 541 arguments.hostnames = [host_info.hostname 542 for host_info in hostname_file_info_list] 543 arguments.host_info_list = hostname_file_info_list 544 else: 545 arguments.host_info_list = [] 546 return arguments 547