1#!/usr/bin/env vpython3 2# 3# Copyright 2018 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import argparse 8import collections 9import json 10import logging 11import os 12import re 13import shutil 14import signal 15import socket 16import sys 17import tempfile 18 19# The following non-std imports are fetched via vpython. See the list at 20# //.vpython3 21import dateutil.parser # pylint: disable=import-error 22import jsonlines # pylint: disable=import-error 23import psutil # pylint: disable=import-error 24 25CHROMIUM_SRC_PATH = os.path.abspath( 26 os.path.join(os.path.dirname(__file__), '..', '..')) 27 28# Use the android test-runner's gtest results support library for generating 29# output json ourselves. 30sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'android')) 31from pylib.base import base_test_result # pylint: disable=import-error 32from pylib.results import json_results # pylint: disable=import-error 33 34sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'util')) 35# TODO(crbug.com/40259280): Re-enable the 'no-name-in-module' check. 36from lib.results import result_sink # pylint: disable=import-error,no-name-in-module 37 38import subprocess # pylint: disable=import-error,wrong-import-order 39 40DEFAULT_CROS_CACHE = os.path.abspath( 41 os.path.join(CHROMIUM_SRC_PATH, 'build', 'cros_cache')) 42CHROMITE_PATH = os.path.abspath( 43 os.path.join(CHROMIUM_SRC_PATH, 'third_party', 'chromite')) 44CROS_RUN_TEST_PATH = os.path.abspath( 45 os.path.join(CHROMITE_PATH, 'bin', 'cros_run_test')) 46 47LACROS_LAUNCHER_SCRIPT_PATH = os.path.abspath( 48 os.path.join(CHROMIUM_SRC_PATH, 'build', 'lacros', 49 'mojo_connection_lacros_launcher.py')) 50 51# This is a special hostname that resolves to a different DUT in the lab 52# depending on which lab machine you're on. 53LAB_DUT_HOSTNAME = 'variable_chromeos_device_hostname' 54 55SYSTEM_LOG_LOCATIONS = [ 56 '/home/chronos/crash/', 57 '/var/log/chrome/', 58 '/var/log/messages', 59 '/var/log/ui/', 60 '/var/log/lacros/', 61] 62 63TAST_DEBUG_DOC = 'https://bit.ly/2LgvIXz' 64 65 66class TestFormatError(Exception): 67 pass 68 69 70class RemoteTest: 71 72 # This is a basic shell script that can be appended to in order to invoke the 73 # test on the device. 74 BASIC_SHELL_SCRIPT = [ 75 '#!/bin/sh', 76 77 # /home and /tmp are mounted with "noexec" in the device, but some of our 78 # tools and tests use those dirs as a workspace (eg: vpython downloads 79 # python binaries to ~/.vpython-root and /tmp/vpython_bootstrap). 80 # /usr/local/tmp doesn't have this restriction, so change the location of 81 # the home and temp dirs for the duration of the test. 82 'export HOME=/usr/local/tmp', 83 'export TMPDIR=/usr/local/tmp', 84 ] 85 86 def __init__(self, args, unknown_args): 87 self._additional_args = unknown_args 88 self._path_to_outdir = args.path_to_outdir 89 self._test_launcher_summary_output = args.test_launcher_summary_output 90 self._logs_dir = args.logs_dir 91 self._use_vm = args.use_vm 92 self._rdb_client = result_sink.TryInitClient() 93 94 self._retries = 0 95 self._timeout = None 96 self._test_launcher_shard_index = args.test_launcher_shard_index 97 self._test_launcher_total_shards = args.test_launcher_total_shards 98 99 # The location on disk of a shell script that can be optionally used to 100 # invoke the test on the device. If it's not set, we assume self._test_cmd 101 # contains the test invocation. 102 self._on_device_script = None 103 104 self._test_cmd = [ 105 CROS_RUN_TEST_PATH, 106 '--board', 107 args.board, 108 '--cache-dir', 109 args.cros_cache, 110 ] 111 if args.use_vm: 112 self._test_cmd += [ 113 '--start', 114 # Don't persist any filesystem changes after the VM shutsdown. 115 '--copy-on-write', 116 ] 117 else: 118 if args.fetch_cros_hostname: 119 self._test_cmd += ['--device', get_cros_hostname()] 120 else: 121 self._test_cmd += [ 122 '--device', args.device if args.device else LAB_DUT_HOSTNAME 123 ] 124 125 if args.logs_dir: 126 for log in SYSTEM_LOG_LOCATIONS: 127 self._test_cmd += ['--results-src', log] 128 self._test_cmd += [ 129 '--results-dest-dir', 130 os.path.join(args.logs_dir, 'system_logs') 131 ] 132 if args.flash: 133 self._test_cmd += ['--flash'] 134 if args.public_image: 135 self._test_cmd += ['--public-image'] 136 137 self._test_env = setup_env() 138 139 @property 140 def suite_name(self): 141 raise NotImplementedError('Child classes need to define suite name.') 142 143 @property 144 def test_cmd(self): 145 return self._test_cmd 146 147 def write_test_script_to_disk(self, script_contents): 148 # Since we're using an on_device_script to invoke the test, we'll need to 149 # set cwd. 150 self._test_cmd += [ 151 '--remote-cmd', 152 '--cwd', 153 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH), 154 ] 155 logging.info('Running the following command on the device:') 156 logging.info('\n%s', '\n'.join(script_contents)) 157 fd, tmp_path = tempfile.mkstemp(suffix='.sh', dir=self._path_to_outdir) 158 os.fchmod(fd, 0o755) 159 with os.fdopen(fd, 'w') as f: 160 f.write('\n'.join(script_contents) + '\n') 161 return tmp_path 162 163 def write_runtime_files_to_disk(self, runtime_files): 164 logging.info('Writing runtime files to disk.') 165 fd, tmp_path = tempfile.mkstemp(suffix='.txt', dir=self._path_to_outdir) 166 os.fchmod(fd, 0o755) 167 with os.fdopen(fd, 'w') as f: 168 f.write('\n'.join(runtime_files) + '\n') 169 return tmp_path 170 171 def run_test(self): 172 # Traps SIGTERM and kills all child processes of cros_run_test when it's 173 # caught. This will allow us to capture logs from the device if a test hangs 174 # and gets timeout-killed by swarming. See also: 175 # https://chromium.googlesource.com/infra/luci/luci-py/+/main/appengine/swarming/doc/Bot.md#graceful-termination_aka-the-sigterm-and-sigkill-dance 176 test_proc = None 177 178 def _kill_child_procs(trapped_signal, _): 179 logging.warning('Received signal %d. Killing child processes of test.', 180 trapped_signal) 181 if not test_proc or not test_proc.pid: 182 # This shouldn't happen? 183 logging.error('Test process not running.') 184 return 185 for child in psutil.Process(test_proc.pid).children(): 186 logging.warning('Killing process %s', child) 187 child.kill() 188 189 signal.signal(signal.SIGTERM, _kill_child_procs) 190 191 for i in range(self._retries + 1): 192 logging.info('########################################') 193 logging.info('Test attempt #%d', i) 194 logging.info('########################################') 195 test_proc = subprocess.Popen( 196 self._test_cmd, 197 stdout=sys.stdout, 198 stderr=sys.stderr, 199 env=self._test_env) 200 try: 201 test_proc.wait(timeout=self._timeout) 202 except subprocess.TimeoutExpired: # pylint: disable=no-member 203 logging.error('Test timed out. Sending SIGTERM.') 204 # SIGTERM the proc and wait 10s for it to close. 205 test_proc.terminate() 206 try: 207 test_proc.wait(timeout=10) 208 except subprocess.TimeoutExpired: # pylint: disable=no-member 209 # If it hasn't closed in 10s, SIGKILL it. 210 logging.error('Test did not exit in time. Sending SIGKILL.') 211 test_proc.kill() 212 test_proc.wait() 213 logging.info('Test exitted with %d.', test_proc.returncode) 214 if test_proc.returncode == 0: 215 break 216 217 self.post_run(test_proc.returncode) 218 # Allow post_run to override test proc return code. (Useful when the host 219 # side Tast bin returns 0 even for failed tests.) 220 return test_proc.returncode 221 222 def post_run(self, _): 223 if self._on_device_script: 224 os.remove(self._on_device_script) 225 226 @staticmethod 227 def get_artifacts(path): 228 """Crawls a given directory for file artifacts to attach to a test. 229 230 Args: 231 path: Path to a directory to search for artifacts. 232 Returns: 233 A dict mapping name of the artifact to its absolute filepath. 234 """ 235 artifacts = {} 236 for dirpath, _, filenames in os.walk(path): 237 for f in filenames: 238 artifact_path = os.path.join(dirpath, f) 239 artifact_id = os.path.relpath(artifact_path, path) 240 # Some artifacts will have non-Latin characters in the filename, eg: 241 # 'ui_tree_Chinese Pinyin-你好.txt'. ResultDB's API rejects such 242 # characters as an artifact ID, so force the file name down into ascii. 243 # For more info, see: 244 # https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/resultdb/proto/v1/artifact.proto;drc=3bff13b8037ca76ec19f9810033d914af7ec67cb;l=46 245 artifact_id = artifact_id.encode('ascii', 'replace').decode() 246 artifact_id = artifact_id.replace('\\', '?') 247 artifacts[artifact_id] = { 248 'filePath': artifact_path, 249 } 250 return artifacts 251 252 253class TastTest(RemoteTest): 254 255 def __init__(self, args, unknown_args): 256 super().__init__(args, unknown_args) 257 258 self._suite_name = args.suite_name 259 self._tast_vars = args.tast_vars 260 self._tast_retries = args.tast_retries 261 self._tests = args.tests 262 # The CQ passes in '--gtest_filter' when specifying tests to skip. Store it 263 # here and parse it later to integrate it into Tast executions. 264 self._gtest_style_filter = args.gtest_filter 265 self._attr_expr = args.attr_expr 266 self._should_strip = args.strip_chrome 267 self._deploy_lacros = args.deploy_lacros 268 self._deploy_chrome = args.deploy_chrome 269 270 if not self._logs_dir: 271 # The host-side Tast bin returns 0 when tests fail, so we need to capture 272 # and parse its json results to reliably determine if tests fail. 273 raise TestFormatError( 274 'When using the host-side Tast bin, "--logs-dir" must be passed in ' 275 'order to parse its results.') 276 277 # If the first test filter is negative, it should be safe to assume all of 278 # them are, so just test the first filter. 279 if self._gtest_style_filter and self._gtest_style_filter[0] == '-': 280 raise TestFormatError('Negative test filters not supported for Tast.') 281 282 @property 283 def suite_name(self): 284 return self._suite_name 285 286 def build_test_command(self): 287 unsupported_args = [ 288 '--test-launcher-retry-limit', 289 '--test-launcher-batch-limit', 290 '--gtest_repeat', 291 ] 292 for unsupported_arg in unsupported_args: 293 if any(arg.startswith(unsupported_arg) for arg in self._additional_args): 294 logging.info( 295 '%s not supported for Tast tests. The arg will be ignored.', 296 unsupported_arg) 297 self._additional_args = [ 298 arg for arg in self._additional_args 299 if not arg.startswith(unsupported_arg) 300 ] 301 302 # Lacros deployment mounts itself by default. 303 if self._deploy_lacros: 304 self._test_cmd.extend([ 305 '--deploy-lacros', '--lacros-launcher-script', 306 LACROS_LAUNCHER_SCRIPT_PATH 307 ]) 308 if self._deploy_chrome: 309 self._test_cmd.extend(['--deploy', '--mount']) 310 else: 311 self._test_cmd.extend(['--deploy', '--mount']) 312 self._test_cmd += [ 313 '--build-dir', 314 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH) 315 ] + self._additional_args 316 317 # Capture tast's results in the logs dir as well. 318 if self._logs_dir: 319 self._test_cmd += [ 320 '--results-dir', 321 self._logs_dir, 322 ] 323 self._test_cmd += [ 324 '--tast-total-shards=%d' % self._test_launcher_total_shards, 325 '--tast-shard-index=%d' % self._test_launcher_shard_index, 326 ] 327 # If we're using a test filter, replace the contents of the Tast 328 # conditional with a long list of "name:test" expressions, one for each 329 # test in the filter. 330 if self._gtest_style_filter: 331 if self._attr_expr or self._tests: 332 logging.warning( 333 'Presence of --gtest_filter will cause the specified Tast expr' 334 ' or test list to be ignored.') 335 names = [] 336 for test in self._gtest_style_filter.split(':'): 337 names.append('"name:%s"' % test) 338 self._attr_expr = '(' + ' || '.join(names) + ')' 339 340 if self._attr_expr: 341 # Don't use shlex.quote() here. Something funky happens with the arg 342 # as it gets passed down from cros_run_test to tast. (Tast picks up the 343 # escaping single quotes and complains that the attribute expression 344 # "must be within parentheses".) 345 self._test_cmd.append('--tast=%s' % self._attr_expr) 346 else: 347 self._test_cmd.append('--tast') 348 self._test_cmd.extend(self._tests) 349 350 for v in self._tast_vars or []: 351 self._test_cmd.extend(['--tast-var', v]) 352 353 if self._tast_retries: 354 self._test_cmd.append('--tast-retries=%d' % self._tast_retries) 355 356 # Mounting ash-chrome gives it enough disk space to not need stripping, 357 # but only for one not instrumented with code coverage. 358 # Lacros uses --nostrip by default, so there is no need to specify. 359 if not self._deploy_lacros and not self._should_strip: 360 self._test_cmd.append('--nostrip') 361 362 def post_run(self, return_code): 363 tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl') 364 if not os.path.exists(tast_results_path): 365 logging.error( 366 'Tast results not found at %s. Falling back to generic result ' 367 'reporting.', tast_results_path) 368 return super().post_run(return_code) 369 370 # See the link below for the format of the results: 371 # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/cmd/tast/run#TestResult 372 with jsonlines.open(tast_results_path) as reader: 373 tast_results = collections.deque(reader) 374 375 suite_results = base_test_result.TestRunResults() 376 for test in tast_results: 377 errors = test['errors'] 378 start, end = test['start'], test['end'] 379 # Use dateutil to parse the timestamps since datetime can't handle 380 # nanosecond precision. 381 duration = dateutil.parser.parse(end) - dateutil.parser.parse(start) 382 # If the duration is negative, Tast has likely reported an incorrect 383 # duration. See https://issuetracker.google.com/issues/187973541. Round 384 # up to 0 in that case to avoid confusing RDB. 385 duration_ms = max(duration.total_seconds() * 1000, 0) 386 if bool(test['skipReason']): 387 result = base_test_result.ResultType.SKIP 388 elif errors: 389 result = base_test_result.ResultType.FAIL 390 else: 391 result = base_test_result.ResultType.PASS 392 primary_error_message = None 393 error_log = '' 394 if errors: 395 # See the link below for the format of these errors: 396 # https://source.chromium.org/chromiumos/chromiumos/codesearch/+/main:src/platform/tast/src/chromiumos/tast/cmd/tast/internal/run/resultsjson/resultsjson.go 397 primary_error_message = errors[0]['reason'] 398 for err in errors: 399 error_log += err['stack'] + '\n' 400 debug_link = ("If you're unsure why this test failed, consult the steps " 401 'outlined <a href="%s">here</a>.' % TAST_DEBUG_DOC) 402 base_result = base_test_result.BaseTestResult( 403 test['name'], result, duration=duration_ms, log=error_log) 404 suite_results.AddResult(base_result) 405 self._maybe_handle_perf_results(test['name']) 406 407 if self._rdb_client: 408 # Walk the contents of the test's "outDir" and atttach any file found 409 # inside as an RDB 'artifact'. (This could include system logs, screen 410 # shots, etc.) 411 artifacts = self.get_artifacts(test['outDir']) 412 html_artifact = debug_link 413 if result == base_test_result.ResultType.SKIP: 414 html_artifact = 'Test was skipped because: ' + test['skipReason'] 415 self._rdb_client.Post( 416 test['name'], 417 result, 418 duration_ms, 419 error_log, 420 None, 421 artifacts=artifacts, 422 failure_reason=primary_error_message, 423 html_artifact=html_artifact) 424 425 if self._rdb_client and self._logs_dir: 426 # Attach artifacts from the device that don't apply to a single test. 427 artifacts = self.get_artifacts( 428 os.path.join(self._logs_dir, 'system_logs')) 429 artifacts.update( 430 self.get_artifacts(os.path.join(self._logs_dir, 'crashes'))) 431 self._rdb_client.ReportInvocationLevelArtifacts(artifacts) 432 433 if self._test_launcher_summary_output: 434 with open(self._test_launcher_summary_output, 'w') as f: 435 json.dump(json_results.GenerateResultsDict([suite_results]), f) 436 437 if not suite_results.DidRunPass(): 438 return 1 439 if return_code: 440 logging.warning( 441 'No failed tests found, but exit code of %d was returned from ' 442 'cros_run_test.', return_code) 443 return return_code 444 return 0 445 446 def _maybe_handle_perf_results(self, test_name): 447 """Prepares any perf results from |test_name| for process_perf_results. 448 449 - process_perf_results looks for top level directories containing a 450 perf_results.json file and a test_results.json file. The directory names 451 are used as the benchmark names. 452 - If a perf_results.json or results-chart.json file exists in the 453 |test_name| results directory, a top level directory is created and the 454 perf results file is copied to perf_results.json. 455 - A trivial test_results.json file is also created to indicate that the test 456 succeeded (this function would not be called otherwise). 457 - When process_perf_results is run, it will find the expected files in the 458 named directory and upload the benchmark results. 459 """ 460 461 perf_results = os.path.join(self._logs_dir, 'tests', test_name, 462 'perf_results.json') 463 # TODO(stevenjb): Remove check for crosbolt results-chart.json file. 464 if not os.path.exists(perf_results): 465 perf_results = os.path.join(self._logs_dir, 'tests', test_name, 466 'results-chart.json') 467 if os.path.exists(perf_results): 468 benchmark_dir = os.path.join(self._logs_dir, test_name) 469 if not os.path.isdir(benchmark_dir): 470 os.makedirs(benchmark_dir) 471 shutil.copyfile(perf_results, 472 os.path.join(benchmark_dir, 'perf_results.json')) 473 # process_perf_results.py expects a test_results.json file. 474 test_results = {'valid': True, 'failures': []} 475 with open(os.path.join(benchmark_dir, 'test_results.json'), 'w') as out: 476 json.dump(test_results, out) 477 478 479class GTestTest(RemoteTest): 480 481 # The following list corresponds to paths that should not be copied over to 482 # the device during tests. In other words, these files are only ever used on 483 # the host. 484 _FILE_IGNORELIST = [ 485 re.compile(r'.*build/android.*'), 486 re.compile(r'.*build/chromeos.*'), 487 re.compile(r'.*build/cros_cache.*'), 488 # The following matches anything under //testing/ that isn't under 489 # //testing/buildbot/filters/. 490 re.compile(r'.*testing/(?!buildbot/filters).*'), 491 re.compile(r'.*third_party/chromite.*'), 492 ] 493 494 def __init__(self, args, unknown_args): 495 super().__init__(args, unknown_args) 496 497 self._test_cmd = ['vpython3'] + self._test_cmd 498 if not args.clean: 499 self._test_cmd += ['--no-clean'] 500 501 self._test_exe = args.test_exe 502 self._runtime_deps_path = args.runtime_deps_path 503 self._vpython_dir = args.vpython_dir 504 505 self._on_device_script = None 506 self._env_vars = args.env_var 507 self._stop_ui = args.stop_ui 508 self._trace_dir = args.trace_dir 509 self._run_test_sudo_helper = args.run_test_sudo_helper 510 self._set_selinux_label = args.set_selinux_label 511 self._use_deployed_dbus_configs = args.use_deployed_dbus_configs 512 513 @property 514 def suite_name(self): 515 return self._test_exe 516 517 def build_test_command(self): 518 # To keep things easy for us, ensure both types of output locations are 519 # the same. 520 if self._test_launcher_summary_output and self._logs_dir: 521 json_out_dir = os.path.dirname(self._test_launcher_summary_output) or '.' 522 if os.path.abspath(json_out_dir) != os.path.abspath(self._logs_dir): 523 raise TestFormatError( 524 '--test-launcher-summary-output and --logs-dir must point to ' 525 'the same directory.') 526 527 if self._test_launcher_summary_output: 528 result_dir, result_file = os.path.split( 529 self._test_launcher_summary_output) 530 # If args.test_launcher_summary_output is a file in cwd, result_dir will 531 # be an empty string, so replace it with '.' when this is the case so 532 # cros_run_test can correctly handle it. 533 if not result_dir: 534 result_dir = '.' 535 device_result_file = '/tmp/%s' % result_file 536 self._test_cmd += [ 537 '--results-src', 538 device_result_file, 539 '--results-dest-dir', 540 result_dir, 541 ] 542 543 if self._trace_dir and self._logs_dir: 544 trace_path = os.path.dirname(self._trace_dir) or '.' 545 if os.path.abspath(trace_path) != os.path.abspath(self._logs_dir): 546 raise TestFormatError( 547 '--trace-dir and --logs-dir must point to the same directory.') 548 549 if self._trace_dir: 550 trace_path, trace_dirname = os.path.split(self._trace_dir) 551 device_trace_dir = '/tmp/%s' % trace_dirname 552 self._test_cmd += [ 553 '--results-src', 554 device_trace_dir, 555 '--results-dest-dir', 556 trace_path, 557 ] 558 559 # Build the shell script that will be used on the device to invoke the test. 560 # Stored here as a list of lines. 561 device_test_script_contents = self.BASIC_SHELL_SCRIPT[:] 562 for var_name, var_val in self._env_vars: 563 device_test_script_contents += ['export %s=%s' % (var_name, var_val)] 564 565 if self._vpython_dir: 566 vpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, 567 'vpython3') 568 cpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, 569 'bin', 'python3') 570 if not os.path.exists(vpython_path) or not os.path.exists(cpython_path): 571 raise TestFormatError( 572 '--vpython-dir must point to a dir with both ' 573 'infra/3pp/tools/cpython3 and infra/tools/luci/vpython3 ' 574 'installed.') 575 vpython_spec_path = os.path.relpath( 576 os.path.join(CHROMIUM_SRC_PATH, '.vpython3'), self._path_to_outdir) 577 # Initialize the vpython cache. This can take 10-20s, and some tests 578 # can't afford to wait that long on the first invocation. 579 device_test_script_contents.extend([ 580 'export PATH=$PWD/%s:$PWD/%s/bin/:$PATH' % 581 (self._vpython_dir, self._vpython_dir), 582 'vpython3 -vpython-spec %s -vpython-tool install' % 583 (vpython_spec_path), 584 ]) 585 586 test_invocation = ('LD_LIBRARY_PATH=./ ./%s --test-launcher-shard-index=%d ' 587 '--test-launcher-total-shards=%d' % 588 (self._test_exe, self._test_launcher_shard_index, 589 self._test_launcher_total_shards)) 590 if self._test_launcher_summary_output: 591 test_invocation += ' --test-launcher-summary-output=%s' % ( 592 device_result_file) 593 594 if self._trace_dir: 595 device_test_script_contents.extend([ 596 'rm -rf %s' % device_trace_dir, 597 'sudo -E -u chronos -- /bin/bash -c "mkdir -p %s"' % device_trace_dir, 598 ]) 599 test_invocation += ' --trace-dir=%s' % device_trace_dir 600 601 if self._run_test_sudo_helper: 602 device_test_script_contents.extend([ 603 'TEST_SUDO_HELPER_PATH=$(mktemp)', 604 './test_sudo_helper.py --socket-path=${TEST_SUDO_HELPER_PATH} &', 605 'TEST_SUDO_HELPER_PID=$!' 606 ]) 607 test_invocation += ( 608 ' --test-sudo-helper-socket-path=${TEST_SUDO_HELPER_PATH}') 609 610 # Append the selinux labels. The 'setfiles' command takes a file with each 611 # line consisting of "<file-regex> <file-type> <new-label>", where '--' is 612 # the type of a regular file. 613 if self._set_selinux_label: 614 for label_pair in self._set_selinux_label: 615 filename, label = label_pair.split('=', 1) 616 specfile = filename + '.specfile' 617 device_test_script_contents.extend([ 618 'echo %s -- %s > %s' % (filename, label, specfile), 619 'setfiles -F %s %s' % (specfile, filename), 620 ]) 621 622 # Mount the deploy dbus config dir on top of chrome's dbus dir. Send SIGHUP 623 # to dbus daemon to reload config from the newly mounted dir. 624 if self._use_deployed_dbus_configs: 625 device_test_script_contents.extend([ 626 'mount --bind ./dbus /opt/google/chrome/dbus', 627 'kill -s HUP $(pgrep dbus)', 628 ]) 629 630 if self._additional_args: 631 test_invocation += ' %s' % ' '.join(self._additional_args) 632 633 if self._stop_ui: 634 device_test_script_contents += [ 635 'stop ui', 636 ] 637 # Send a user activity ping to powerd to ensure the display is on. 638 device_test_script_contents += [ 639 'dbus-send --system --type=method_call' 640 ' --dest=org.chromium.PowerManager /org/chromium/PowerManager' 641 ' org.chromium.PowerManager.HandleUserActivity int32:0' 642 ] 643 # The UI service on the device owns the chronos user session, so shutting 644 # it down as chronos kills the entire execution of the test. So we'll have 645 # to run as root up until the test invocation. 646 test_invocation = ( 647 'sudo -E -u chronos -- /bin/bash -c "%s"' % test_invocation) 648 # And we'll need to chown everything since cros_run_test's "--as-chronos" 649 # option normally does that for us. 650 device_test_script_contents.append('chown -R chronos: ../..') 651 else: 652 self._test_cmd += [ 653 # Some tests fail as root, so run as the less privileged user 654 # 'chronos'. 655 '--as-chronos', 656 ] 657 658 device_test_script_contents.append(test_invocation) 659 device_test_script_contents.append('TEST_RETURN_CODE=$?') 660 661 # (Re)start ui after all tests are done. This is for developer convenienve. 662 # Without this, the device would remain in a black screen which looks like 663 # powered off. 664 if self._stop_ui: 665 device_test_script_contents += [ 666 'start ui', 667 ] 668 669 # Stop the crosier helper. 670 if self._run_test_sudo_helper: 671 device_test_script_contents.extend([ 672 'pkill -P $TEST_SUDO_HELPER_PID', 673 'kill $TEST_SUDO_HELPER_PID', 674 'unlink ${TEST_SUDO_HELPER_PATH}', 675 ]) 676 677 # Undo the dbus config mount and reload dbus config. 678 if self._use_deployed_dbus_configs: 679 device_test_script_contents.extend([ 680 'umount /opt/google/chrome/dbus', 681 'kill -s HUP $(pgrep dbus)', 682 ]) 683 684 # This command should always be the last bash commandline so infra can 685 # correctly get the error code from test invocations. 686 device_test_script_contents.append('exit $TEST_RETURN_CODE') 687 688 self._on_device_script = self.write_test_script_to_disk( 689 device_test_script_contents) 690 691 runtime_files = [os.path.relpath(self._on_device_script)] 692 runtime_files += self._read_runtime_files() 693 if self._vpython_dir: 694 # --vpython-dir is relative to the out dir, but --files-from expects paths 695 # relative to src dir, so fix the path up a bit. 696 runtime_files.append( 697 os.path.relpath( 698 os.path.abspath( 699 os.path.join(self._path_to_outdir, self._vpython_dir)), 700 CHROMIUM_SRC_PATH)) 701 702 self._test_cmd.extend( 703 ['--files-from', 704 self.write_runtime_files_to_disk(runtime_files)]) 705 706 self._test_cmd += [ 707 '--', 708 './' + os.path.relpath(self._on_device_script, self._path_to_outdir) 709 ] 710 711 def _read_runtime_files(self): 712 if not self._runtime_deps_path: 713 return [] 714 715 abs_runtime_deps_path = os.path.abspath( 716 os.path.join(self._path_to_outdir, self._runtime_deps_path)) 717 with open(abs_runtime_deps_path) as runtime_deps_file: 718 files = [l.strip() for l in runtime_deps_file if l] 719 rel_file_paths = [] 720 for f in files: 721 rel_file_path = os.path.relpath( 722 os.path.abspath(os.path.join(self._path_to_outdir, f))) 723 if not any(regex.match(rel_file_path) for regex in self._FILE_IGNORELIST): 724 rel_file_paths.append(rel_file_path) 725 return rel_file_paths 726 727 def post_run(self, _): 728 if self._on_device_script: 729 os.remove(self._on_device_script) 730 731 if self._test_launcher_summary_output and self._rdb_client: 732 logging.error('Native ResultDB integration is not supported for GTests. ' 733 'Upload results via result_adapter instead. ' 734 'See crbug.com/1330441.') 735 736 737def device_test(args, unknown_args): 738 # cros_run_test has trouble with relative paths that go up directories, 739 # so cd to src/, which should be the root of all data deps. 740 os.chdir(CHROMIUM_SRC_PATH) 741 742 # TODO: Remove the above when depot_tool's pylint is updated to include the 743 # fix to https://github.com/PyCQA/pylint/issues/710. 744 if args.test_type == 'tast': 745 test = TastTest(args, unknown_args) 746 else: 747 test = GTestTest(args, unknown_args) 748 749 test.build_test_command() 750 logging.info('Running the following command on the device:') 751 logging.info(' '.join(test.test_cmd)) 752 753 return test.run_test() 754 755 756def host_cmd(args, cmd_args): 757 if not cmd_args: 758 raise TestFormatError('Must specify command to run on the host.') 759 if args.deploy_chrome and not args.path_to_outdir: 760 raise TestFormatError( 761 '--path-to-outdir must be specified if --deploy-chrome is passed.') 762 763 cros_run_test_cmd = [ 764 CROS_RUN_TEST_PATH, 765 '--board', 766 args.board, 767 '--cache-dir', 768 os.path.join(CHROMIUM_SRC_PATH, args.cros_cache), 769 ] 770 if args.use_vm: 771 cros_run_test_cmd += [ 772 '--start', 773 # Don't persist any filesystem changes after the VM shutsdown. 774 '--copy-on-write', 775 ] 776 else: 777 if args.fetch_cros_hostname: 778 cros_run_test_cmd += ['--device', get_cros_hostname()] 779 else: 780 cros_run_test_cmd += [ 781 '--device', args.device if args.device else LAB_DUT_HOSTNAME 782 ] 783 if args.verbose: 784 cros_run_test_cmd.append('--debug') 785 if args.flash: 786 cros_run_test_cmd.append('--flash') 787 if args.public_image: 788 cros_run_test_cmd += ['--public-image'] 789 790 if args.logs_dir: 791 for log in SYSTEM_LOG_LOCATIONS: 792 cros_run_test_cmd += ['--results-src', log] 793 cros_run_test_cmd += [ 794 '--results-dest-dir', 795 os.path.join(args.logs_dir, 'system_logs') 796 ] 797 798 test_env = setup_env() 799 if args.deploy_chrome or args.deploy_lacros: 800 if args.deploy_lacros: 801 cros_run_test_cmd.extend([ 802 '--deploy-lacros', '--lacros-launcher-script', 803 LACROS_LAUNCHER_SCRIPT_PATH 804 ]) 805 if args.deploy_chrome: 806 # Mounting ash-chrome gives it enough disk space to not need stripping 807 # most of the time. 808 cros_run_test_cmd.extend(['--deploy', '--mount']) 809 else: 810 # Mounting ash-chrome gives it enough disk space to not need stripping 811 # most of the time. 812 cros_run_test_cmd.extend(['--deploy', '--mount']) 813 814 if not args.strip_chrome: 815 cros_run_test_cmd.append('--nostrip') 816 817 cros_run_test_cmd += [ 818 '--build-dir', 819 os.path.join(CHROMIUM_SRC_PATH, args.path_to_outdir) 820 ] 821 822 cros_run_test_cmd += [ 823 '--host-cmd', 824 '--', 825 ] + cmd_args 826 827 logging.info('Running the following command:') 828 logging.info(' '.join(cros_run_test_cmd)) 829 830 return subprocess.call( 831 cros_run_test_cmd, stdout=sys.stdout, stderr=sys.stderr, env=test_env) 832 833 834def get_cros_hostname_from_bot_id(bot_id): 835 """Parse hostname from a chromeos-swarming bot id.""" 836 for prefix in ['cros-', 'crossk-']: 837 if bot_id.startswith(prefix): 838 return bot_id[len(prefix):] 839 return bot_id 840 841 842def get_cros_hostname(): 843 """Fetch bot_id from env var and parse hostname.""" 844 845 # In chromeos-swarming, we can extract hostname from bot ID, since 846 # bot ID is formatted as "{prefix}{hostname}". 847 bot_id = os.environ.get('SWARMING_BOT_ID') 848 if bot_id: 849 return get_cros_hostname_from_bot_id(bot_id) 850 851 logging.warning( 852 'Attempted to read from SWARMING_BOT_ID env var and it was' 853 ' not defined. Will set %s as device instead.', LAB_DUT_HOSTNAME) 854 return LAB_DUT_HOSTNAME 855 856 857def setup_env(): 858 """Returns a copy of the current env with some needed vars added.""" 859 env = os.environ.copy() 860 # Some chromite scripts expect chromite/bin to be on PATH. 861 env['PATH'] = env['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin') 862 # deploy_chrome needs a set of GN args used to build chrome to determine if 863 # certain libraries need to be pushed to the device. It looks for the args via 864 # an env var. To trigger the default deploying behavior, give it a dummy set 865 # of args. 866 # TODO(crbug.com/40567963): Make the GN-dependent deps controllable via cmd 867 # line args. 868 if not env.get('GN_ARGS'): 869 env['GN_ARGS'] = 'enable_nacl = true' 870 if not env.get('USE'): 871 env['USE'] = 'highdpi' 872 return env 873 874 875def add_common_args(*parsers): 876 for parser in parsers: 877 parser.add_argument('--verbose', '-v', action='store_true') 878 parser.add_argument( 879 '--board', type=str, required=True, help='Type of CrOS device.') 880 parser.add_argument( 881 '--deploy-chrome', 882 action='store_true', 883 help='Will deploy a locally built ash-chrome binary to the device ' 884 'before running the host-cmd.') 885 parser.add_argument( 886 '--deploy-lacros', action='store_true', help='Deploy a lacros-chrome.') 887 parser.add_argument( 888 '--cros-cache', 889 type=str, 890 default=DEFAULT_CROS_CACHE, 891 help='Path to cros cache.') 892 parser.add_argument( 893 '--path-to-outdir', 894 type=str, 895 required=True, 896 help='Path to output directory, all of whose contents will be ' 897 'deployed to the device.') 898 parser.add_argument( 899 '--runtime-deps-path', 900 type=str, 901 help='Runtime data dependency file from GN.') 902 parser.add_argument( 903 '--vpython-dir', 904 type=str, 905 help='Location on host of a directory containing a vpython binary to ' 906 'deploy to the device before the test starts. The location of ' 907 'this dir will be added onto PATH in the device. WARNING: The ' 908 'arch of the device might not match the arch of the host, so ' 909 'avoid using "${platform}" when downloading vpython via CIPD.') 910 parser.add_argument( 911 '--logs-dir', 912 type=str, 913 dest='logs_dir', 914 help='Will copy everything under /var/log/ from the device after the ' 915 'test into the specified dir.') 916 # Shard args are parsed here since we might also specify them via env vars. 917 parser.add_argument( 918 '--test-launcher-shard-index', 919 type=int, 920 default=os.environ.get('GTEST_SHARD_INDEX', 0), 921 help='Index of the external shard to run.') 922 parser.add_argument( 923 '--test-launcher-total-shards', 924 type=int, 925 default=os.environ.get('GTEST_TOTAL_SHARDS', 1), 926 help='Total number of external shards.') 927 parser.add_argument( 928 '--flash', 929 action='store_true', 930 help='Will flash the device to the current SDK version before running ' 931 'the test.') 932 parser.add_argument( 933 '--no-flash', 934 action='store_false', 935 dest='flash', 936 help='Will not flash the device before running the test.') 937 parser.add_argument( 938 '--public-image', 939 action='store_true', 940 help='Will flash a public "full" image to the device.') 941 parser.add_argument( 942 '--magic-vm-cache', 943 help='Path to the magic CrOS VM cache dir. See the comment above ' 944 '"magic_cros_vm_cache" in mixins.pyl for more info.') 945 946 vm_or_device_group = parser.add_mutually_exclusive_group() 947 vm_or_device_group.add_argument( 948 '--use-vm', 949 action='store_true', 950 help='Will run the test in the VM instead of a device.') 951 vm_or_device_group.add_argument( 952 '--device', 953 type=str, 954 help='Hostname (or IP) of device to run the test on. This arg is not ' 955 'required if --use-vm is set.') 956 vm_or_device_group.add_argument( 957 '--fetch-cros-hostname', 958 action='store_true', 959 help='Will extract device hostname from the SWARMING_BOT_ID env var if ' 960 'running on ChromeOS Swarming.') 961 962def main(): 963 parser = argparse.ArgumentParser() 964 subparsers = parser.add_subparsers(dest='test_type') 965 # Host-side test args. 966 host_cmd_parser = subparsers.add_parser( 967 'host-cmd', 968 help='Runs a host-side test. Pass the host-side command to run after ' 969 '"--". If --use-vm is passed, hostname and port for the device ' 970 'will be 127.0.0.1:9222.') 971 host_cmd_parser.set_defaults(func=host_cmd) 972 host_cmd_parser.add_argument( 973 '--strip-chrome', 974 action='store_true', 975 help='Strips symbols from ash-chrome or lacros-chrome before deploying ' 976 ' to the device.') 977 978 gtest_parser = subparsers.add_parser( 979 'gtest', help='Runs a device-side gtest.') 980 gtest_parser.set_defaults(func=device_test) 981 gtest_parser.add_argument( 982 '--test-exe', 983 type=str, 984 required=True, 985 help='Path to test executable to run inside the device.') 986 987 # GTest args. Some are passed down to the test binary in the device. Others 988 # are parsed here since they might need tweaking or special handling. 989 gtest_parser.add_argument( 990 '--test-launcher-summary-output', 991 type=str, 992 help='When set, will pass the same option down to the test and retrieve ' 993 'its result file at the specified location.') 994 gtest_parser.add_argument( 995 '--stop-ui', 996 action='store_true', 997 help='Will stop the UI service in the device before running the test. ' 998 'Also start the UI service after all tests are done.') 999 gtest_parser.add_argument( 1000 '--trace-dir', 1001 type=str, 1002 help='When set, will pass down to the test to generate the trace and ' 1003 'retrieve the trace files to the specified location.') 1004 gtest_parser.add_argument( 1005 '--env-var', 1006 nargs=2, 1007 action='append', 1008 default=[], 1009 help='Env var to set on the device for the duration of the test. ' 1010 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify ' 1011 'multiple times for more than one var.') 1012 gtest_parser.add_argument( 1013 '--run-test-sudo-helper', 1014 action='store_true', 1015 help='When set, will run test_sudo_helper before the test and stop it ' 1016 'after test finishes.') 1017 gtest_parser.add_argument( 1018 "--no-clean", 1019 action="store_false", 1020 dest="clean", 1021 default=True, 1022 help="Do not clean up the deployed files after running the test. " 1023 "Only supported for --remote-cmd tests") 1024 gtest_parser.add_argument( 1025 '--set-selinux-label', 1026 action='append', 1027 default=[], 1028 help='Set the selinux label for a file before running. The format is:\n' 1029 ' --set-selinux-label=<filename>=<label>\n' 1030 'So:\n' 1031 ' --set-selinux-label=my_test=u:r:cros_foo_label:s0\n' 1032 'You can specify it more than one time to set multiple files tags.') 1033 gtest_parser.add_argument( 1034 '--use-deployed-dbus-configs', 1035 action='store_true', 1036 help='When set, will bind mount deployed dbus config to chrome dbus dir ' 1037 'and ask dbus daemon to reload config before running tests.') 1038 1039 # Tast test args. 1040 # pylint: disable=line-too-long 1041 tast_test_parser = subparsers.add_parser( 1042 'tast', 1043 help='Runs a device-side set of Tast tests. For more details, see: ' 1044 'https://chromium.googlesource.com/chromiumos/platform/tast/+/main/docs/running_tests.md' 1045 ) 1046 tast_test_parser.set_defaults(func=device_test) 1047 tast_test_parser.add_argument( 1048 '--suite-name', 1049 type=str, 1050 required=True, 1051 help='Name to apply to the set of Tast tests to run. This has no effect ' 1052 'on what is executed, but is used mainly for test results reporting ' 1053 'and tracking (eg: flakiness dashboard).') 1054 tast_test_parser.add_argument( 1055 '--test-launcher-summary-output', 1056 type=str, 1057 help='Generates a simple GTest-style JSON result file for the test run.') 1058 tast_test_parser.add_argument( 1059 '--attr-expr', 1060 type=str, 1061 help='A boolean expression whose matching tests will run ' 1062 '(eg: ("dep:chrome")).') 1063 tast_test_parser.add_argument( 1064 '--strip-chrome', 1065 action='store_true', 1066 help='Strips symbols from ash-chrome before deploying to the device.') 1067 tast_test_parser.add_argument( 1068 '--tast-var', 1069 action='append', 1070 dest='tast_vars', 1071 help='Runtime variables for Tast tests, and the format are expected to ' 1072 'be "key=value" pairs.') 1073 tast_test_parser.add_argument( 1074 '--tast-retries', 1075 type=int, 1076 dest='tast_retries', 1077 help='Number of retries for failed Tast tests on the same DUT.') 1078 tast_test_parser.add_argument( 1079 '--test', 1080 '-t', 1081 action='append', 1082 dest='tests', 1083 help='A Tast test to run in the device (eg: "login.Chrome").') 1084 tast_test_parser.add_argument( 1085 '--gtest_filter', 1086 type=str, 1087 help="Similar to GTest's arg of the same name, this will filter out the " 1088 "specified tests from the Tast run. However, due to the nature of Tast's " 1089 'cmd-line API, this will overwrite the value(s) of "--test" above.') 1090 1091 add_common_args(gtest_parser, tast_test_parser, host_cmd_parser) 1092 args, unknown_args = parser.parse_known_args() 1093 # Re-add N-1 -v/--verbose flags to the args we'll pass to whatever we are 1094 # running. The assumption is that only one verbosity incrase would be meant 1095 # for this script since it's a boolean value instead of increasing verbosity 1096 # with more instances. 1097 verbose_flags = [a for a in sys.argv if a in ('-v', '--verbose')] 1098 if verbose_flags: 1099 unknown_args += verbose_flags[1:] 1100 1101 logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) 1102 1103 if not args.use_vm and not args.device and not args.fetch_cros_hostname: 1104 logging.warning( 1105 'The test runner is now assuming running in the lab environment, if ' 1106 'this is unintentional, please re-invoke the test runner with the ' 1107 '"--use-vm" arg if using a VM, otherwise use the "--device=<DUT>" arg ' 1108 'to specify a DUT.') 1109 1110 # If we're not running on a VM, but haven't specified a hostname, assume 1111 # we're on a lab bot and are trying to run a test on a lab DUT. See if the 1112 # magic lab DUT hostname resolves to anything. (It will in the lab and will 1113 # not on dev machines.) 1114 try: 1115 socket.getaddrinfo(LAB_DUT_HOSTNAME, None) 1116 except socket.gaierror: 1117 logging.error('The default lab DUT hostname of %s is unreachable.', 1118 LAB_DUT_HOSTNAME) 1119 return 1 1120 1121 if args.flash and args.public_image: 1122 # The flashing tools depend on being unauthenticated with GS when flashing 1123 # public images, so make sure the env var GS uses to locate its creds is 1124 # unset in that case. 1125 os.environ.pop('BOTO_CONFIG', None) 1126 1127 if args.magic_vm_cache: 1128 full_vm_cache_path = os.path.join(CHROMIUM_SRC_PATH, args.magic_vm_cache) 1129 if os.path.exists(full_vm_cache_path): 1130 with open(os.path.join(full_vm_cache_path, 'swarming.txt'), 'w') as f: 1131 f.write('non-empty file to make swarming persist this cache') 1132 1133 return args.func(args, unknown_args) 1134 1135 1136if __name__ == '__main__': 1137 sys.exit(main()) 1138