1#!/usr/bin/env vpython3 2# 3# Copyright 2018 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import argparse 8import collections 9import json 10import logging 11import os 12import re 13import shutil 14import signal 15import socket 16import sys 17import tempfile 18 19# The following non-std imports are fetched via vpython. See the list at 20# //.vpython3 21import dateutil.parser # pylint: disable=import-error 22import jsonlines # pylint: disable=import-error 23import psutil # pylint: disable=import-error 24 25CHROMIUM_SRC_PATH = os.path.abspath( 26 os.path.join(os.path.dirname(__file__), '..', '..')) 27 28# Use the android test-runner's gtest results support library for generating 29# output json ourselves. 30sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'android')) 31from pylib.base import base_test_result # pylint: disable=import-error 32from pylib.results import json_results # pylint: disable=import-error 33 34sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'util')) 35# TODO(crbug.com/1421441): Re-enable the 'no-name-in-module' check. 36from lib.results import result_sink # pylint: disable=import-error,no-name-in-module 37 38import subprocess # pylint: disable=import-error,wrong-import-order 39 40DEFAULT_CROS_CACHE = os.path.abspath( 41 os.path.join(CHROMIUM_SRC_PATH, 'build', 'cros_cache')) 42CHROMITE_PATH = os.path.abspath( 43 os.path.join(CHROMIUM_SRC_PATH, 'third_party', 'chromite')) 44CROS_RUN_TEST_PATH = os.path.abspath( 45 os.path.join(CHROMITE_PATH, 'bin', 'cros_run_test')) 46 47LACROS_LAUNCHER_SCRIPT_PATH = os.path.abspath( 48 os.path.join(CHROMIUM_SRC_PATH, 'build', 'lacros', 49 'mojo_connection_lacros_launcher.py')) 50 51# This is a special hostname that resolves to a different DUT in the lab 52# depending on which lab machine you're on. 53LAB_DUT_HOSTNAME = 'variable_chromeos_device_hostname' 54 55SYSTEM_LOG_LOCATIONS = [ 56 '/home/chronos/crash/', 57 '/var/log/chrome/', 58 '/var/log/messages', 59 '/var/log/ui/', 60] 61 62TAST_DEBUG_DOC = 'https://bit.ly/2LgvIXz' 63 64 65class TestFormatError(Exception): 66 pass 67 68 69class RemoteTest: 70 71 # This is a basic shell script that can be appended to in order to invoke the 72 # test on the device. 73 BASIC_SHELL_SCRIPT = [ 74 '#!/bin/sh', 75 76 # /home and /tmp are mounted with "noexec" in the device, but some of our 77 # tools and tests use those dirs as a workspace (eg: vpython downloads 78 # python binaries to ~/.vpython-root and /tmp/vpython_bootstrap). 79 # /usr/local/tmp doesn't have this restriction, so change the location of 80 # the home and temp dirs for the duration of the test. 81 'export HOME=/usr/local/tmp', 82 'export TMPDIR=/usr/local/tmp', 83 ] 84 85 def __init__(self, args, unknown_args): 86 self._additional_args = unknown_args 87 self._path_to_outdir = args.path_to_outdir 88 self._test_launcher_summary_output = args.test_launcher_summary_output 89 self._logs_dir = args.logs_dir 90 self._use_vm = args.use_vm 91 self._rdb_client = result_sink.TryInitClient() 92 93 self._retries = 0 94 self._timeout = None 95 self._test_launcher_shard_index = args.test_launcher_shard_index 96 self._test_launcher_total_shards = args.test_launcher_total_shards 97 98 # The location on disk of a shell script that can be optionally used to 99 # invoke the test on the device. If it's not set, we assume self._test_cmd 100 # contains the test invocation. 101 self._on_device_script = None 102 103 self._test_cmd = [ 104 CROS_RUN_TEST_PATH, 105 '--board', 106 args.board, 107 '--cache-dir', 108 args.cros_cache, 109 ] 110 if args.use_vm: 111 self._test_cmd += [ 112 '--start', 113 # Don't persist any filesystem changes after the VM shutsdown. 114 '--copy-on-write', 115 ] 116 else: 117 self._test_cmd += [ 118 '--device', args.device if args.device else LAB_DUT_HOSTNAME 119 ] 120 if args.logs_dir: 121 for log in SYSTEM_LOG_LOCATIONS: 122 self._test_cmd += ['--results-src', log] 123 self._test_cmd += [ 124 '--results-dest-dir', 125 os.path.join(args.logs_dir, 'system_logs') 126 ] 127 if args.flash: 128 self._test_cmd += ['--flash'] 129 if args.public_image: 130 self._test_cmd += ['--public-image'] 131 132 self._test_env = setup_env() 133 134 @property 135 def suite_name(self): 136 raise NotImplementedError('Child classes need to define suite name.') 137 138 @property 139 def test_cmd(self): 140 return self._test_cmd 141 142 def write_test_script_to_disk(self, script_contents): 143 # Since we're using an on_device_script to invoke the test, we'll need to 144 # set cwd. 145 self._test_cmd += [ 146 '--remote-cmd', 147 '--cwd', 148 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH), 149 ] 150 logging.info('Running the following command on the device:') 151 logging.info('\n%s', '\n'.join(script_contents)) 152 fd, tmp_path = tempfile.mkstemp(suffix='.sh', dir=self._path_to_outdir) 153 os.fchmod(fd, 0o755) 154 with os.fdopen(fd, 'w') as f: 155 f.write('\n'.join(script_contents) + '\n') 156 return tmp_path 157 158 def run_test(self): 159 # Traps SIGTERM and kills all child processes of cros_run_test when it's 160 # caught. This will allow us to capture logs from the device if a test hangs 161 # and gets timeout-killed by swarming. See also: 162 # https://chromium.googlesource.com/infra/luci/luci-py/+/main/appengine/swarming/doc/Bot.md#graceful-termination_aka-the-sigterm-and-sigkill-dance 163 test_proc = None 164 165 def _kill_child_procs(trapped_signal, _): 166 logging.warning('Received signal %d. Killing child processes of test.', 167 trapped_signal) 168 if not test_proc or not test_proc.pid: 169 # This shouldn't happen? 170 logging.error('Test process not running.') 171 return 172 for child in psutil.Process(test_proc.pid).children(): 173 logging.warning('Killing process %s', child) 174 child.kill() 175 176 signal.signal(signal.SIGTERM, _kill_child_procs) 177 178 for i in range(self._retries + 1): 179 logging.info('########################################') 180 logging.info('Test attempt #%d', i) 181 logging.info('########################################') 182 test_proc = subprocess.Popen( 183 self._test_cmd, 184 stdout=sys.stdout, 185 stderr=sys.stderr, 186 env=self._test_env) 187 try: 188 test_proc.wait(timeout=self._timeout) 189 except subprocess.TimeoutExpired: # pylint: disable=no-member 190 logging.error('Test timed out. Sending SIGTERM.') 191 # SIGTERM the proc and wait 10s for it to close. 192 test_proc.terminate() 193 try: 194 test_proc.wait(timeout=10) 195 except subprocess.TimeoutExpired: # pylint: disable=no-member 196 # If it hasn't closed in 10s, SIGKILL it. 197 logging.error('Test did not exit in time. Sending SIGKILL.') 198 test_proc.kill() 199 test_proc.wait() 200 logging.info('Test exitted with %d.', test_proc.returncode) 201 if test_proc.returncode == 0: 202 break 203 204 self.post_run(test_proc.returncode) 205 # Allow post_run to override test proc return code. (Useful when the host 206 # side Tast bin returns 0 even for failed tests.) 207 return test_proc.returncode 208 209 def post_run(self, _): 210 if self._on_device_script: 211 os.remove(self._on_device_script) 212 213 @staticmethod 214 def get_artifacts(path): 215 """Crawls a given directory for file artifacts to attach to a test. 216 217 Args: 218 path: Path to a directory to search for artifacts. 219 Returns: 220 A dict mapping name of the artifact to its absolute filepath. 221 """ 222 artifacts = {} 223 for dirpath, _, filenames in os.walk(path): 224 for f in filenames: 225 artifact_path = os.path.join(dirpath, f) 226 artifact_id = os.path.relpath(artifact_path, path) 227 # Some artifacts will have non-Latin characters in the filename, eg: 228 # 'ui_tree_Chinese Pinyin-你好.txt'. ResultDB's API rejects such 229 # characters as an artifact ID, so force the file name down into ascii. 230 # For more info, see: 231 # https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/resultdb/proto/v1/artifact.proto;drc=3bff13b8037ca76ec19f9810033d914af7ec67cb;l=46 232 artifact_id = artifact_id.encode('ascii', 'replace').decode() 233 artifact_id = artifact_id.replace('\\', '?') 234 artifacts[artifact_id] = { 235 'filePath': artifact_path, 236 } 237 return artifacts 238 239 240class TastTest(RemoteTest): 241 242 def __init__(self, args, unknown_args): 243 super().__init__(args, unknown_args) 244 245 self._suite_name = args.suite_name 246 self._tast_vars = args.tast_vars 247 self._tast_retries = args.tast_retries 248 self._tests = args.tests 249 # The CQ passes in '--gtest_filter' when specifying tests to skip. Store it 250 # here and parse it later to integrate it into Tast executions. 251 self._gtest_style_filter = args.gtest_filter 252 self._attr_expr = args.attr_expr 253 self._should_strip = args.strip_chrome 254 self._deploy_lacros = args.deploy_lacros 255 self._deploy_chrome = args.deploy_chrome 256 257 if not self._logs_dir: 258 # The host-side Tast bin returns 0 when tests fail, so we need to capture 259 # and parse its json results to reliably determine if tests fail. 260 raise TestFormatError( 261 'When using the host-side Tast bin, "--logs-dir" must be passed in ' 262 'order to parse its results.') 263 264 # If the first test filter is negative, it should be safe to assume all of 265 # them are, so just test the first filter. 266 if self._gtest_style_filter and self._gtest_style_filter[0] == '-': 267 raise TestFormatError('Negative test filters not supported for Tast.') 268 269 @property 270 def suite_name(self): 271 return self._suite_name 272 273 def build_test_command(self): 274 unsupported_args = [ 275 '--test-launcher-retry-limit', 276 '--test-launcher-batch-limit', 277 '--gtest_repeat', 278 ] 279 for unsupported_arg in unsupported_args: 280 if any(arg.startswith(unsupported_arg) for arg in self._additional_args): 281 logging.info( 282 '%s not supported for Tast tests. The arg will be ignored.', 283 unsupported_arg) 284 self._additional_args = [ 285 arg for arg in self._additional_args 286 if not arg.startswith(unsupported_arg) 287 ] 288 289 # Lacros deployment mounts itself by default. 290 if self._deploy_lacros: 291 self._test_cmd.extend([ 292 '--deploy-lacros', '--lacros-launcher-script', 293 LACROS_LAUNCHER_SCRIPT_PATH 294 ]) 295 if self._deploy_chrome: 296 self._test_cmd.extend(['--deploy', '--mount']) 297 else: 298 self._test_cmd.extend(['--deploy', '--mount']) 299 self._test_cmd += [ 300 '--build-dir', 301 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH) 302 ] + self._additional_args 303 304 # Capture tast's results in the logs dir as well. 305 if self._logs_dir: 306 self._test_cmd += [ 307 '--results-dir', 308 self._logs_dir, 309 ] 310 self._test_cmd += [ 311 '--tast-total-shards=%d' % self._test_launcher_total_shards, 312 '--tast-shard-index=%d' % self._test_launcher_shard_index, 313 ] 314 # If we're using a test filter, replace the contents of the Tast 315 # conditional with a long list of "name:test" expressions, one for each 316 # test in the filter. 317 if self._gtest_style_filter: 318 if self._attr_expr or self._tests: 319 logging.warning( 320 'Presence of --gtest_filter will cause the specified Tast expr' 321 ' or test list to be ignored.') 322 names = [] 323 for test in self._gtest_style_filter.split(':'): 324 names.append('"name:%s"' % test) 325 self._attr_expr = '(' + ' || '.join(names) + ')' 326 327 if self._attr_expr: 328 # Don't use shlex.quote() here. Something funky happens with the arg 329 # as it gets passed down from cros_run_test to tast. (Tast picks up the 330 # escaping single quotes and complains that the attribute expression 331 # "must be within parentheses".) 332 self._test_cmd.append('--tast=%s' % self._attr_expr) 333 else: 334 self._test_cmd.append('--tast') 335 self._test_cmd.extend(self._tests) 336 337 for v in self._tast_vars or []: 338 self._test_cmd.extend(['--tast-var', v]) 339 340 if self._tast_retries: 341 self._test_cmd.append('--tast-retries=%d' % self._tast_retries) 342 343 # Mounting ash-chrome gives it enough disk space to not need stripping, 344 # but only for one not instrumented with code coverage. 345 # Lacros uses --nostrip by default, so there is no need to specify. 346 if not self._deploy_lacros and not self._should_strip: 347 self._test_cmd.append('--nostrip') 348 349 def post_run(self, return_code): 350 tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl') 351 if not os.path.exists(tast_results_path): 352 logging.error( 353 'Tast results not found at %s. Falling back to generic result ' 354 'reporting.', tast_results_path) 355 return super().post_run(return_code) 356 357 # See the link below for the format of the results: 358 # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/cmd/tast/run#TestResult 359 with jsonlines.open(tast_results_path) as reader: 360 tast_results = collections.deque(reader) 361 362 suite_results = base_test_result.TestRunResults() 363 for test in tast_results: 364 errors = test['errors'] 365 start, end = test['start'], test['end'] 366 # Use dateutil to parse the timestamps since datetime can't handle 367 # nanosecond precision. 368 duration = dateutil.parser.parse(end) - dateutil.parser.parse(start) 369 # If the duration is negative, Tast has likely reported an incorrect 370 # duration. See https://issuetracker.google.com/issues/187973541. Round 371 # up to 0 in that case to avoid confusing RDB. 372 duration_ms = max(duration.total_seconds() * 1000, 0) 373 if bool(test['skipReason']): 374 result = base_test_result.ResultType.SKIP 375 elif errors: 376 result = base_test_result.ResultType.FAIL 377 else: 378 result = base_test_result.ResultType.PASS 379 primary_error_message = None 380 error_log = '' 381 if errors: 382 # See the link below for the format of these errors: 383 # https://source.chromium.org/chromiumos/chromiumos/codesearch/+/main:src/platform/tast/src/chromiumos/tast/cmd/tast/internal/run/resultsjson/resultsjson.go 384 primary_error_message = errors[0]['reason'] 385 for err in errors: 386 error_log += err['stack'] + '\n' 387 debug_link = ("If you're unsure why this test failed, consult the steps " 388 'outlined <a href="%s">here</a>.' % TAST_DEBUG_DOC) 389 base_result = base_test_result.BaseTestResult( 390 test['name'], result, duration=duration_ms, log=error_log) 391 suite_results.AddResult(base_result) 392 self._maybe_handle_perf_results(test['name']) 393 394 if self._rdb_client: 395 # Walk the contents of the test's "outDir" and atttach any file found 396 # inside as an RDB 'artifact'. (This could include system logs, screen 397 # shots, etc.) 398 artifacts = self.get_artifacts(test['outDir']) 399 html_artifact = debug_link 400 if result == base_test_result.ResultType.SKIP: 401 html_artifact = 'Test was skipped because: ' + test['skipReason'] 402 self._rdb_client.Post( 403 test['name'], 404 result, 405 duration_ms, 406 error_log, 407 None, 408 artifacts=artifacts, 409 failure_reason=primary_error_message, 410 html_artifact=html_artifact) 411 412 if self._rdb_client and self._logs_dir: 413 # Attach artifacts from the device that don't apply to a single test. 414 artifacts = self.get_artifacts( 415 os.path.join(self._logs_dir, 'system_logs')) 416 artifacts.update( 417 self.get_artifacts(os.path.join(self._logs_dir, 'crashes'))) 418 self._rdb_client.ReportInvocationLevelArtifacts(artifacts) 419 420 if self._test_launcher_summary_output: 421 with open(self._test_launcher_summary_output, 'w') as f: 422 json.dump(json_results.GenerateResultsDict([suite_results]), f) 423 424 if not suite_results.DidRunPass(): 425 return 1 426 if return_code: 427 logging.warning( 428 'No failed tests found, but exit code of %d was returned from ' 429 'cros_run_test.', return_code) 430 return return_code 431 return 0 432 433 def _maybe_handle_perf_results(self, test_name): 434 """Prepares any perf results from |test_name| for process_perf_results. 435 436 - process_perf_results looks for top level directories containing a 437 perf_results.json file and a test_results.json file. The directory names 438 are used as the benchmark names. 439 - If a perf_results.json or results-chart.json file exists in the 440 |test_name| results directory, a top level directory is created and the 441 perf results file is copied to perf_results.json. 442 - A trivial test_results.json file is also created to indicate that the test 443 succeeded (this function would not be called otherwise). 444 - When process_perf_results is run, it will find the expected files in the 445 named directory and upload the benchmark results. 446 """ 447 448 perf_results = os.path.join(self._logs_dir, 'tests', test_name, 449 'perf_results.json') 450 # TODO(stevenjb): Remove check for crosbolt results-chart.json file. 451 if not os.path.exists(perf_results): 452 perf_results = os.path.join(self._logs_dir, 'tests', test_name, 453 'results-chart.json') 454 if os.path.exists(perf_results): 455 benchmark_dir = os.path.join(self._logs_dir, test_name) 456 if not os.path.isdir(benchmark_dir): 457 os.makedirs(benchmark_dir) 458 shutil.copyfile(perf_results, 459 os.path.join(benchmark_dir, 'perf_results.json')) 460 # process_perf_results.py expects a test_results.json file. 461 test_results = {'valid': True, 'failures': []} 462 with open(os.path.join(benchmark_dir, 'test_results.json'), 'w') as out: 463 json.dump(test_results, out) 464 465 466class GTestTest(RemoteTest): 467 468 # The following list corresponds to paths that should not be copied over to 469 # the device during tests. In other words, these files are only ever used on 470 # the host. 471 _FILE_IGNORELIST = [ 472 re.compile(r'.*build/android.*'), 473 re.compile(r'.*build/chromeos.*'), 474 re.compile(r'.*build/cros_cache.*'), 475 # The following matches anything under //testing/ that isn't under 476 # //testing/buildbot/filters/. 477 re.compile(r'.*testing/(?!buildbot/filters).*'), 478 re.compile(r'.*third_party/chromite.*'), 479 ] 480 481 def __init__(self, args, unknown_args): 482 super().__init__(args, unknown_args) 483 484 self._test_cmd = ['vpython3'] + self._test_cmd 485 if not args.clean: 486 self._test_cmd += ['--no-clean'] 487 488 self._test_exe = args.test_exe 489 self._runtime_deps_path = args.runtime_deps_path 490 self._vpython_dir = args.vpython_dir 491 492 self._on_device_script = None 493 self._env_vars = args.env_var 494 self._stop_ui = args.stop_ui 495 self._trace_dir = args.trace_dir 496 self._run_test_sudo_helper = args.run_test_sudo_helper 497 self._set_selinux_label = args.set_selinux_label 498 499 @property 500 def suite_name(self): 501 return self._test_exe 502 503 def build_test_command(self): 504 # To keep things easy for us, ensure both types of output locations are 505 # the same. 506 if self._test_launcher_summary_output and self._logs_dir: 507 json_out_dir = os.path.dirname(self._test_launcher_summary_output) or '.' 508 if os.path.abspath(json_out_dir) != os.path.abspath(self._logs_dir): 509 raise TestFormatError( 510 '--test-launcher-summary-output and --logs-dir must point to ' 511 'the same directory.') 512 513 if self._test_launcher_summary_output: 514 result_dir, result_file = os.path.split( 515 self._test_launcher_summary_output) 516 # If args.test_launcher_summary_output is a file in cwd, result_dir will 517 # be an empty string, so replace it with '.' when this is the case so 518 # cros_run_test can correctly handle it. 519 if not result_dir: 520 result_dir = '.' 521 device_result_file = '/tmp/%s' % result_file 522 self._test_cmd += [ 523 '--results-src', 524 device_result_file, 525 '--results-dest-dir', 526 result_dir, 527 ] 528 529 if self._trace_dir and self._logs_dir: 530 trace_path = os.path.dirname(self._trace_dir) or '.' 531 if os.path.abspath(trace_path) != os.path.abspath(self._logs_dir): 532 raise TestFormatError( 533 '--trace-dir and --logs-dir must point to the same directory.') 534 535 if self._trace_dir: 536 trace_path, trace_dirname = os.path.split(self._trace_dir) 537 device_trace_dir = '/tmp/%s' % trace_dirname 538 self._test_cmd += [ 539 '--results-src', 540 device_trace_dir, 541 '--results-dest-dir', 542 trace_path, 543 ] 544 545 # Build the shell script that will be used on the device to invoke the test. 546 # Stored here as a list of lines. 547 device_test_script_contents = self.BASIC_SHELL_SCRIPT[:] 548 for var_name, var_val in self._env_vars: 549 device_test_script_contents += ['export %s=%s' % (var_name, var_val)] 550 551 if self._vpython_dir: 552 vpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, 553 'vpython3') 554 cpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, 555 'bin', 'python3') 556 if not os.path.exists(vpython_path) or not os.path.exists(cpython_path): 557 raise TestFormatError( 558 '--vpython-dir must point to a dir with both ' 559 'infra/3pp/tools/cpython3 and infra/tools/luci/vpython installed.') 560 vpython_spec_path = os.path.relpath( 561 os.path.join(CHROMIUM_SRC_PATH, '.vpython3'), self._path_to_outdir) 562 # Initialize the vpython cache. This can take 10-20s, and some tests 563 # can't afford to wait that long on the first invocation. 564 device_test_script_contents.extend([ 565 'export PATH=$PWD/%s:$PWD/%s/bin/:$PATH' % 566 (self._vpython_dir, self._vpython_dir), 567 'vpython3 -vpython-spec %s -vpython-tool install' % 568 (vpython_spec_path), 569 ]) 570 571 test_invocation = ('LD_LIBRARY_PATH=./ ./%s --test-launcher-shard-index=%d ' 572 '--test-launcher-total-shards=%d' % 573 (self._test_exe, self._test_launcher_shard_index, 574 self._test_launcher_total_shards)) 575 if self._test_launcher_summary_output: 576 test_invocation += ' --test-launcher-summary-output=%s' % ( 577 device_result_file) 578 579 if self._trace_dir: 580 device_test_script_contents.extend([ 581 'rm -rf %s' % device_trace_dir, 582 'sudo -E -u chronos -- /bin/bash -c "mkdir -p %s"' % device_trace_dir, 583 ]) 584 test_invocation += ' --trace-dir=%s' % device_trace_dir 585 586 if self._run_test_sudo_helper: 587 device_test_script_contents.extend([ 588 'TEST_SUDO_HELPER_PATH=$(mktemp)', 589 './test_sudo_helper.py --socket-path=${TEST_SUDO_HELPER_PATH} &', 590 'TEST_SUDO_HELPER_PID=$!' 591 ]) 592 test_invocation += ( 593 ' --test-sudo-helper-socket-path=${TEST_SUDO_HELPER_PATH}') 594 595 # Append the selinux labels. The 'setfiles' command takes a file with each 596 # line consisting of "<file-regex> <file-type> <new-label>", where '--' is 597 # the type of a regular file. 598 if self._set_selinux_label: 599 for label_pair in self._set_selinux_label: 600 filename, label = label_pair.split('=', 1) 601 specfile = filename + '.specfile' 602 device_test_script_contents.extend([ 603 'echo %s -- %s > %s' % (filename, label, specfile), 604 'setfiles -F %s %s' % (specfile, filename), 605 ]) 606 607 if self._additional_args: 608 test_invocation += ' %s' % ' '.join(self._additional_args) 609 610 if self._stop_ui: 611 device_test_script_contents += [ 612 'stop ui', 613 ] 614 # Send a user activity ping to powerd to ensure the display is on. 615 device_test_script_contents += [ 616 'dbus-send --system --type=method_call' 617 ' --dest=org.chromium.PowerManager /org/chromium/PowerManager' 618 ' org.chromium.PowerManager.HandleUserActivity int32:0' 619 ] 620 # The UI service on the device owns the chronos user session, so shutting 621 # it down as chronos kills the entire execution of the test. So we'll have 622 # to run as root up until the test invocation. 623 test_invocation = ( 624 'sudo -E -u chronos -- /bin/bash -c "%s"' % test_invocation) 625 # And we'll need to chown everything since cros_run_test's "--as-chronos" 626 # option normally does that for us. 627 device_test_script_contents.append('chown -R chronos: ../..') 628 else: 629 self._test_cmd += [ 630 # Some tests fail as root, so run as the less privileged user 631 # 'chronos'. 632 '--as-chronos', 633 ] 634 635 device_test_script_contents.append(test_invocation) 636 637 # (Re)start ui after all tests are done. This is for developer convenienve. 638 # Without this, the device would remain in a black screen which looks like 639 # powered off. 640 if self._stop_ui: 641 device_test_script_contents += [ 642 'start ui', 643 ] 644 645 # Stop the crosier helper. 646 if self._run_test_sudo_helper: 647 device_test_script_contents.extend([ 648 'pkill -P $TEST_SUDO_HELPER_PID', 649 'kill $TEST_SUDO_HELPER_PID', 650 'unlink ${TEST_SUDO_HELPER_PATH}', 651 ]) 652 653 self._on_device_script = self.write_test_script_to_disk( 654 device_test_script_contents) 655 656 runtime_files = [os.path.relpath(self._on_device_script)] 657 runtime_files += self._read_runtime_files() 658 if self._vpython_dir: 659 # --vpython-dir is relative to the out dir, but --files expects paths 660 # relative to src dir, so fix the path up a bit. 661 runtime_files.append( 662 os.path.relpath( 663 os.path.abspath( 664 os.path.join(self._path_to_outdir, self._vpython_dir)), 665 CHROMIUM_SRC_PATH)) 666 667 for f in runtime_files: 668 self._test_cmd.extend(['--files', f]) 669 670 self._test_cmd += [ 671 '--', 672 './' + os.path.relpath(self._on_device_script, self._path_to_outdir) 673 ] 674 675 def _read_runtime_files(self): 676 if not self._runtime_deps_path: 677 return [] 678 679 abs_runtime_deps_path = os.path.abspath( 680 os.path.join(self._path_to_outdir, self._runtime_deps_path)) 681 with open(abs_runtime_deps_path) as runtime_deps_file: 682 files = [l.strip() for l in runtime_deps_file if l] 683 rel_file_paths = [] 684 for f in files: 685 rel_file_path = os.path.relpath( 686 os.path.abspath(os.path.join(self._path_to_outdir, f))) 687 if not any(regex.match(rel_file_path) for regex in self._FILE_IGNORELIST): 688 rel_file_paths.append(rel_file_path) 689 return rel_file_paths 690 691 def post_run(self, _): 692 if self._on_device_script: 693 os.remove(self._on_device_script) 694 695 if self._test_launcher_summary_output and self._rdb_client: 696 logging.error('Native ResultDB integration is not supported for GTests. ' 697 'Upload results via result_adapter instead. ' 698 'See crbug.com/1330441.') 699 700 701def device_test(args, unknown_args): 702 # cros_run_test has trouble with relative paths that go up directories, 703 # so cd to src/, which should be the root of all data deps. 704 os.chdir(CHROMIUM_SRC_PATH) 705 706 # TODO: Remove the above when depot_tool's pylint is updated to include the 707 # fix to https://github.com/PyCQA/pylint/issues/710. 708 if args.test_type == 'tast': 709 test = TastTest(args, unknown_args) 710 else: 711 test = GTestTest(args, unknown_args) 712 713 test.build_test_command() 714 logging.info('Running the following command on the device:') 715 logging.info(' '.join(test.test_cmd)) 716 717 return test.run_test() 718 719 720def host_cmd(args, cmd_args): 721 if not cmd_args: 722 raise TestFormatError('Must specify command to run on the host.') 723 if args.deploy_chrome and not args.path_to_outdir: 724 raise TestFormatError( 725 '--path-to-outdir must be specified if --deploy-chrome is passed.') 726 727 cros_run_test_cmd = [ 728 CROS_RUN_TEST_PATH, 729 '--board', 730 args.board, 731 '--cache-dir', 732 os.path.join(CHROMIUM_SRC_PATH, args.cros_cache), 733 ] 734 if args.use_vm: 735 cros_run_test_cmd += [ 736 '--start', 737 # Don't persist any filesystem changes after the VM shutsdown. 738 '--copy-on-write', 739 ] 740 else: 741 cros_run_test_cmd += [ 742 '--device', args.device if args.device else LAB_DUT_HOSTNAME 743 ] 744 if args.verbose: 745 cros_run_test_cmd.append('--debug') 746 if args.flash: 747 cros_run_test_cmd.append('--flash') 748 if args.public_image: 749 cros_run_test_cmd += ['--public-image'] 750 751 if args.logs_dir: 752 for log in SYSTEM_LOG_LOCATIONS: 753 cros_run_test_cmd += ['--results-src', log] 754 cros_run_test_cmd += [ 755 '--results-dest-dir', 756 os.path.join(args.logs_dir, 'system_logs') 757 ] 758 759 test_env = setup_env() 760 if args.deploy_chrome or args.deploy_lacros: 761 if args.deploy_lacros: 762 cros_run_test_cmd.extend([ 763 '--deploy-lacros', '--lacros-launcher-script', 764 LACROS_LAUNCHER_SCRIPT_PATH 765 ]) 766 if args.deploy_chrome: 767 # Mounting ash-chrome gives it enough disk space to not need stripping 768 # most of the time. 769 cros_run_test_cmd.extend(['--deploy', '--mount']) 770 else: 771 # Mounting ash-chrome gives it enough disk space to not need stripping 772 # most of the time. 773 cros_run_test_cmd.extend(['--deploy', '--mount']) 774 775 if not args.strip_chrome: 776 cros_run_test_cmd.append('--nostrip') 777 778 cros_run_test_cmd += [ 779 '--build-dir', 780 os.path.join(CHROMIUM_SRC_PATH, args.path_to_outdir) 781 ] 782 783 cros_run_test_cmd += [ 784 '--host-cmd', 785 '--', 786 ] + cmd_args 787 788 logging.info('Running the following command:') 789 logging.info(' '.join(cros_run_test_cmd)) 790 791 return subprocess.call( 792 cros_run_test_cmd, stdout=sys.stdout, stderr=sys.stderr, env=test_env) 793 794 795def setup_env(): 796 """Returns a copy of the current env with some needed vars added.""" 797 env = os.environ.copy() 798 # Some chromite scripts expect chromite/bin to be on PATH. 799 env['PATH'] = env['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin') 800 # deploy_chrome needs a set of GN args used to build chrome to determine if 801 # certain libraries need to be pushed to the device. It looks for the args via 802 # an env var. To trigger the default deploying behavior, give it a dummy set 803 # of args. 804 # TODO(crbug.com/823996): Make the GN-dependent deps controllable via cmd 805 # line args. 806 if not env.get('GN_ARGS'): 807 env['GN_ARGS'] = 'enable_nacl = true' 808 if not env.get('USE'): 809 env['USE'] = 'highdpi' 810 return env 811 812 813def add_common_args(*parsers): 814 for parser in parsers: 815 parser.add_argument('--verbose', '-v', action='store_true') 816 parser.add_argument( 817 '--board', type=str, required=True, help='Type of CrOS device.') 818 parser.add_argument( 819 '--deploy-chrome', 820 action='store_true', 821 help='Will deploy a locally built ash-chrome binary to the device ' 822 'before running the host-cmd.') 823 parser.add_argument( 824 '--deploy-lacros', action='store_true', help='Deploy a lacros-chrome.') 825 parser.add_argument( 826 '--cros-cache', 827 type=str, 828 default=DEFAULT_CROS_CACHE, 829 help='Path to cros cache.') 830 parser.add_argument( 831 '--path-to-outdir', 832 type=str, 833 required=True, 834 help='Path to output directory, all of whose contents will be ' 835 'deployed to the device.') 836 parser.add_argument( 837 '--runtime-deps-path', 838 type=str, 839 help='Runtime data dependency file from GN.') 840 parser.add_argument( 841 '--vpython-dir', 842 type=str, 843 help='Location on host of a directory containing a vpython binary to ' 844 'deploy to the device before the test starts. The location of ' 845 'this dir will be added onto PATH in the device. WARNING: The ' 846 'arch of the device might not match the arch of the host, so ' 847 'avoid using "${platform}" when downloading vpython via CIPD.') 848 parser.add_argument( 849 '--logs-dir', 850 type=str, 851 dest='logs_dir', 852 help='Will copy everything under /var/log/ from the device after the ' 853 'test into the specified dir.') 854 # Shard args are parsed here since we might also specify them via env vars. 855 parser.add_argument( 856 '--test-launcher-shard-index', 857 type=int, 858 default=os.environ.get('GTEST_SHARD_INDEX', 0), 859 help='Index of the external shard to run.') 860 parser.add_argument( 861 '--test-launcher-total-shards', 862 type=int, 863 default=os.environ.get('GTEST_TOTAL_SHARDS', 1), 864 help='Total number of external shards.') 865 parser.add_argument( 866 '--flash', 867 action='store_true', 868 help='Will flash the device to the current SDK version before running ' 869 'the test.') 870 parser.add_argument( 871 '--no-flash', 872 action='store_false', 873 dest='flash', 874 help='Will not flash the device before running the test.') 875 parser.add_argument( 876 '--public-image', 877 action='store_true', 878 help='Will flash a public "full" image to the device.') 879 parser.add_argument( 880 '--magic-vm-cache', 881 help='Path to the magic CrOS VM cache dir. See the comment above ' 882 '"magic_cros_vm_cache" in mixins.pyl for more info.') 883 884 vm_or_device_group = parser.add_mutually_exclusive_group() 885 vm_or_device_group.add_argument( 886 '--use-vm', 887 action='store_true', 888 help='Will run the test in the VM instead of a device.') 889 vm_or_device_group.add_argument( 890 '--device', 891 type=str, 892 help='Hostname (or IP) of device to run the test on. This arg is not ' 893 'required if --use-vm is set.') 894 895 896def main(): 897 parser = argparse.ArgumentParser() 898 subparsers = parser.add_subparsers(dest='test_type') 899 # Host-side test args. 900 host_cmd_parser = subparsers.add_parser( 901 'host-cmd', 902 help='Runs a host-side test. Pass the host-side command to run after ' 903 '"--". If --use-vm is passed, hostname and port for the device ' 904 'will be 127.0.0.1:9222.') 905 host_cmd_parser.set_defaults(func=host_cmd) 906 host_cmd_parser.add_argument( 907 '--strip-chrome', 908 action='store_true', 909 help='Strips symbols from ash-chrome or lacros-chrome before deploying ' 910 ' to the device.') 911 912 gtest_parser = subparsers.add_parser( 913 'gtest', help='Runs a device-side gtest.') 914 gtest_parser.set_defaults(func=device_test) 915 gtest_parser.add_argument( 916 '--test-exe', 917 type=str, 918 required=True, 919 help='Path to test executable to run inside the device.') 920 921 # GTest args. Some are passed down to the test binary in the device. Others 922 # are parsed here since they might need tweaking or special handling. 923 gtest_parser.add_argument( 924 '--test-launcher-summary-output', 925 type=str, 926 help='When set, will pass the same option down to the test and retrieve ' 927 'its result file at the specified location.') 928 gtest_parser.add_argument( 929 '--stop-ui', 930 action='store_true', 931 help='Will stop the UI service in the device before running the test. ' 932 'Also start the UI service after all tests are done.') 933 gtest_parser.add_argument( 934 '--trace-dir', 935 type=str, 936 help='When set, will pass down to the test to generate the trace and ' 937 'retrieve the trace files to the specified location.') 938 gtest_parser.add_argument( 939 '--env-var', 940 nargs=2, 941 action='append', 942 default=[], 943 help='Env var to set on the device for the duration of the test. ' 944 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify ' 945 'multiple times for more than one var.') 946 gtest_parser.add_argument( 947 '--run-test-sudo-helper', 948 action='store_true', 949 help='When set, will run test_sudo_helper before the test and stop it ' 950 'after test finishes.') 951 gtest_parser.add_argument( 952 "--no-clean", 953 action="store_false", 954 dest="clean", 955 default=True, 956 help="Do not clean up the deployed files after running the test. " 957 "Only supported for --remote-cmd tests") 958 gtest_parser.add_argument( 959 '--set-selinux-label', 960 action='append', 961 default=[], 962 help='Set the selinux label for a file before running. The format is:\n' 963 ' --set-selinux-label=<filename>=<label>\n' 964 'So:\n' 965 ' --set-selinux-label=my_test=u:r:cros_foo_label:s0\n' 966 'You can specify it more than one time to set multiple files tags.') 967 968 # Tast test args. 969 # pylint: disable=line-too-long 970 tast_test_parser = subparsers.add_parser( 971 'tast', 972 help='Runs a device-side set of Tast tests. For more details, see: ' 973 'https://chromium.googlesource.com/chromiumos/platform/tast/+/main/docs/running_tests.md' 974 ) 975 tast_test_parser.set_defaults(func=device_test) 976 tast_test_parser.add_argument( 977 '--suite-name', 978 type=str, 979 required=True, 980 help='Name to apply to the set of Tast tests to run. This has no effect ' 981 'on what is executed, but is used mainly for test results reporting ' 982 'and tracking (eg: flakiness dashboard).') 983 tast_test_parser.add_argument( 984 '--test-launcher-summary-output', 985 type=str, 986 help='Generates a simple GTest-style JSON result file for the test run.') 987 tast_test_parser.add_argument( 988 '--attr-expr', 989 type=str, 990 help='A boolean expression whose matching tests will run ' 991 '(eg: ("dep:chrome")).') 992 tast_test_parser.add_argument( 993 '--strip-chrome', 994 action='store_true', 995 help='Strips symbols from ash-chrome before deploying to the device.') 996 tast_test_parser.add_argument( 997 '--tast-var', 998 action='append', 999 dest='tast_vars', 1000 help='Runtime variables for Tast tests, and the format are expected to ' 1001 'be "key=value" pairs.') 1002 tast_test_parser.add_argument( 1003 '--tast-retries', 1004 type=int, 1005 dest='tast_retries', 1006 help='Number of retries for failed Tast tests on the same DUT.') 1007 tast_test_parser.add_argument( 1008 '--test', 1009 '-t', 1010 action='append', 1011 dest='tests', 1012 help='A Tast test to run in the device (eg: "login.Chrome").') 1013 tast_test_parser.add_argument( 1014 '--gtest_filter', 1015 type=str, 1016 help="Similar to GTest's arg of the same name, this will filter out the " 1017 "specified tests from the Tast run. However, due to the nature of Tast's " 1018 'cmd-line API, this will overwrite the value(s) of "--test" above.') 1019 1020 add_common_args(gtest_parser, tast_test_parser, host_cmd_parser) 1021 args, unknown_args = parser.parse_known_args() 1022 # Re-add N-1 -v/--verbose flags to the args we'll pass to whatever we are 1023 # running. The assumption is that only one verbosity incrase would be meant 1024 # for this script since it's a boolean value instead of increasing verbosity 1025 # with more instances. 1026 verbose_flags = [a for a in sys.argv if a in ('-v', '--verbose')] 1027 if verbose_flags: 1028 unknown_args += verbose_flags[1:] 1029 1030 logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) 1031 1032 if not args.use_vm and not args.device: 1033 logging.warning( 1034 'The test runner is now assuming running in the lab environment, if ' 1035 'this is unintentional, please re-invoke the test runner with the ' 1036 '"--use-vm" arg if using a VM, otherwise use the "--device=<DUT>" arg ' 1037 'to specify a DUT.') 1038 1039 # If we're not running on a VM, but haven't specified a hostname, assume 1040 # we're on a lab bot and are trying to run a test on a lab DUT. See if the 1041 # magic lab DUT hostname resolves to anything. (It will in the lab and will 1042 # not on dev machines.) 1043 try: 1044 socket.getaddrinfo(LAB_DUT_HOSTNAME, None) 1045 except socket.gaierror: 1046 logging.error('The default lab DUT hostname of %s is unreachable.', 1047 LAB_DUT_HOSTNAME) 1048 return 1 1049 1050 if args.flash and args.public_image: 1051 # The flashing tools depend on being unauthenticated with GS when flashing 1052 # public images, so make sure the env var GS uses to locate its creds is 1053 # unset in that case. 1054 os.environ.pop('BOTO_CONFIG', None) 1055 1056 if args.magic_vm_cache: 1057 full_vm_cache_path = os.path.join(CHROMIUM_SRC_PATH, args.magic_vm_cache) 1058 if os.path.exists(full_vm_cache_path): 1059 with open(os.path.join(full_vm_cache_path, 'swarming.txt'), 'w') as f: 1060 f.write('non-empty file to make swarming persist this cache') 1061 1062 return args.func(args, unknown_args) 1063 1064 1065if __name__ == '__main__': 1066 sys.exit(main()) 1067