Lines Matching +full:test +full:- +full:path
4 # Use of this source code is governed by a BSD-style license that can be
19 # The following non-std imports are fetched via vpython. See the list at
21 import dateutil.parser # pylint: disable=import-error
22 import jsonlines # pylint: disable=import-error
23 import psutil # pylint: disable=import-error
25 CHROMIUM_SRC_PATH = os.path.abspath(
26 os.path.join(os.path.dirname(__file__), '..', '..'))
28 # Use the android test-runner's gtest results support library for generating
30 sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'android'))
31 from pylib.base import base_test_result # pylint: disable=import-error
32 from pylib.results import json_results # pylint: disable=import-error
34 sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'util'))
35 # TODO(crbug.com/40259280): Re-enable the 'no-name-in-module' check.
36 from lib.results import result_sink # pylint: disable=import-error,no-name-in-module
38 import subprocess # pylint: disable=import-error,wrong-import-order
40 DEFAULT_CROS_CACHE = os.path.abspath(
41 os.path.join(CHROMIUM_SRC_PATH, 'build', 'cros_cache'))
42 CHROMITE_PATH = os.path.abspath(
43 os.path.join(CHROMIUM_SRC_PATH, 'third_party', 'chromite'))
44 CROS_RUN_TEST_PATH = os.path.abspath(
45 os.path.join(CHROMITE_PATH, 'bin', 'cros_run_test'))
47 LACROS_LAUNCHER_SCRIPT_PATH = os.path.abspath(
48 os.path.join(CHROMIUM_SRC_PATH, 'build', 'lacros',
73 # test on the device.
79 # python binaries to ~/.vpython-root and /tmp/vpython_bootstrap).
81 # the home and temp dirs for the duration of the test.
100 # invoke the test on the device. If it's not set, we assume self._test_cmd
101 # contains the test invocation.
106 '--board',
108 '--cache-dir',
113 '--start',
115 '--copy-on-write',
119 self._test_cmd += ['--device', get_cros_hostname()]
122 '--device', args.device if args.device else LAB_DUT_HOSTNAME
127 self._test_cmd += ['--results-src', log]
129 '--results-dest-dir',
130 os.path.join(args.logs_dir, 'system_logs')
133 self._test_cmd += ['--flash']
135 self._test_cmd += ['--public-image']
148 # Since we're using an on_device_script to invoke the test, we'll need to
151 '--remote-cmd',
152 '--cwd',
153 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH),
173 # caught. This will allow us to capture logs from the device if a test hangs
174 # and gets timeout-killed by swarming. See also:
175 …ooglesource.com/infra/luci/luci-py/+/main/appengine/swarming/doc/Bot.md#graceful-termination_aka-t…
179 logging.warning('Received signal %d. Killing child processes of test.',
183 logging.error('Test process not running.')
193 logging.info('Test attempt #%d', i)
202 except subprocess.TimeoutExpired: # pylint: disable=no-member
203 logging.error('Test timed out. Sending SIGTERM.')
208 except subprocess.TimeoutExpired: # pylint: disable=no-member
210 logging.error('Test did not exit in time. Sending SIGKILL.')
213 logging.info('Test exitted with %d.', test_proc.returncode)
218 # Allow post_run to override test proc return code. (Useful when the host
227 def get_artifacts(path): argument
228 """Crawls a given directory for file artifacts to attach to a test.
231 path: Path to a directory to search for artifacts.
236 for dirpath, _, filenames in os.walk(path):
238 artifact_path = os.path.join(dirpath, f)
239 artifact_id = os.path.relpath(artifact_path, path)
240 # Some artifacts will have non-Latin characters in the filename, eg:
241 # 'ui_tree_Chinese Pinyin-你好.txt'. ResultDB's API rejects such
262 # The CQ passes in '--gtest_filter' when specifying tests to skip. Store it
271 # The host-side Tast bin returns 0 when tests fail, so we need to capture
274 'When using the host-side Tast bin, "--logs-dir" must be passed in '
277 # If the first test filter is negative, it should be safe to assume all of
278 # them are, so just test the first filter.
279 if self._gtest_style_filter and self._gtest_style_filter[0] == '-':
280 raise TestFormatError('Negative test filters not supported for Tast.')
288 '--test-launcher-retry-limit',
289 '--test-launcher-batch-limit',
290 '--gtest_repeat',
305 '--deploy-lacros', '--lacros-launcher-script',
309 self._test_cmd.extend(['--deploy', '--mount'])
311 self._test_cmd.extend(['--deploy', '--mount'])
313 '--build-dir',
314 os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH)
320 '--results-dir',
324 '--tast-total-shards=%d' % self._test_launcher_total_shards,
325 '--tast-shard-index=%d' % self._test_launcher_shard_index,
327 # If we're using a test filter, replace the contents of the Tast
328 # conditional with a long list of "name:test" expressions, one for each
329 # test in the filter.
333 'Presence of --gtest_filter will cause the specified Tast expr'
334 ' or test list to be ignored.')
336 for test in self._gtest_style_filter.split(':'):
337 names.append('"name:%s"' % test)
345 self._test_cmd.append('--tast=%s' % self._attr_expr)
347 self._test_cmd.append('--tast')
351 self._test_cmd.extend(['--tast-var', v])
354 self._test_cmd.append('--tast-retries=%d' % self._tast_retries)
356 # Mounting ash-chrome gives it enough disk space to not need stripping,
358 # Lacros uses --nostrip by default, so there is no need to specify.
360 self._test_cmd.append('--nostrip')
363 tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl')
364 if not os.path.exists(tast_results_path):
376 for test in tast_results:
377 errors = test['errors']
378 start, end = test['start'], test['end']
381 duration = dateutil.parser.parse(end) - dateutil.parser.parse(start)
386 if bool(test['skipReason']):
400 debug_link = ("If you're unsure why this test failed, consult the steps "
403 test['name'], result, duration=duration_ms, log=error_log)
405 self._maybe_handle_perf_results(test['name'])
408 # Walk the contents of the test's "outDir" and atttach any file found
411 artifacts = self.get_artifacts(test['outDir'])
414 html_artifact = 'Test was skipped because: ' + test['skipReason']
416 test['name'],
426 # Attach artifacts from the device that don't apply to a single test.
428 os.path.join(self._logs_dir, 'system_logs'))
430 self.get_artifacts(os.path.join(self._logs_dir, 'crashes')))
449 - process_perf_results looks for top level directories containing a
452 - If a perf_results.json or results-chart.json file exists in the
455 - A trivial test_results.json file is also created to indicate that the test
457 - When process_perf_results is run, it will find the expected files in the
461 perf_results = os.path.join(self._logs_dir, 'tests', test_name,
463 # TODO(stevenjb): Remove check for crosbolt results-chart.json file.
464 if not os.path.exists(perf_results):
465 perf_results = os.path.join(self._logs_dir, 'tests', test_name,
466 'results-chart.json')
467 if os.path.exists(perf_results):
468 benchmark_dir = os.path.join(self._logs_dir, test_name)
469 if not os.path.isdir(benchmark_dir):
472 os.path.join(benchmark_dir, 'perf_results.json'))
475 with open(os.path.join(benchmark_dir, 'test_results.json'), 'w') as out:
499 self._test_cmd += ['--no-clean']
521 json_out_dir = os.path.dirname(self._test_launcher_summary_output) or '.'
522 if os.path.abspath(json_out_dir) != os.path.abspath(self._logs_dir):
524 '--test-launcher-summary-output and --logs-dir must point to '
528 result_dir, result_file = os.path.split(
537 '--results-src',
539 '--results-dest-dir',
544 trace_path = os.path.dirname(self._trace_dir) or '.'
545 if os.path.abspath(trace_path) != os.path.abspath(self._logs_dir):
547 '--trace-dir and --logs-dir must point to the same directory.')
550 trace_path, trace_dirname = os.path.split(self._trace_dir)
553 '--results-src',
555 '--results-dest-dir',
559 # Build the shell script that will be used on the device to invoke the test.
566 vpython_path = os.path.join(self._path_to_outdir, self._vpython_dir,
568 cpython_path = os.path.join(self._path_to_outdir, self._vpython_dir,
570 if not os.path.exists(vpython_path) or not os.path.exists(cpython_path):
572 '--vpython-dir must point to a dir with both '
575 vpython_spec_path = os.path.relpath(
576 os.path.join(CHROMIUM_SRC_PATH, '.vpython3'), self._path_to_outdir)
577 # Initialize the vpython cache. This can take 10-20s, and some tests
580 'export PATH=$PWD/%s:$PWD/%s/bin/:$PATH' %
582 'vpython3 -vpython-spec %s -vpython-tool install' %
586 test_invocation = ('LD_LIBRARY_PATH=./ ./%s --test-launcher-shard-index=%d '
587 '--test-launcher-total-shards=%d' %
591 test_invocation += ' --test-launcher-summary-output=%s' % (
596 'rm -rf %s' % device_trace_dir,
597 'sudo -E -u chronos -- /bin/bash -c "mkdir -p %s"' % device_trace_dir,
599 test_invocation += ' --trace-dir=%s' % device_trace_dir
604 './test_sudo_helper.py --socket-path=${TEST_SUDO_HELPER_PATH} &',
608 ' --test-sudo-helper-socket-path=${TEST_SUDO_HELPER_PATH}')
611 # line consisting of "<file-regex> <file-type> <new-label>", where '--' is
618 'echo %s -- %s > %s' % (filename, label, specfile),
619 'setfiles -F %s %s' % (specfile, filename),
626 'mount --bind ./dbus /opt/google/chrome/dbus',
627 'kill -s HUP $(pgrep dbus)',
639 'dbus-send --system --type=method_call'
640 ' --dest=org.chromium.PowerManager /org/chromium/PowerManager'
644 # it down as chronos kills the entire execution of the test. So we'll have
645 # to run as root up until the test invocation.
647 'sudo -E -u chronos -- /bin/bash -c "%s"' % test_invocation)
648 # And we'll need to chown everything since cros_run_test's "--as-chronos"
650 device_test_script_contents.append('chown -R chronos: ../..')
655 '--as-chronos',
672 'pkill -P $TEST_SUDO_HELPER_PID',
681 'kill -s HUP $(pgrep dbus)',
685 # correctly get the error code from test invocations.
691 runtime_files = [os.path.relpath(self._on_device_script)]
694 # --vpython-dir is relative to the out dir, but --files-from expects paths
695 # relative to src dir, so fix the path up a bit.
697 os.path.relpath(
698 os.path.abspath(
699 os.path.join(self._path_to_outdir, self._vpython_dir)),
703 ['--files-from',
707 '--',
708 './' + os.path.relpath(self._on_device_script, self._path_to_outdir)
715 abs_runtime_deps_path = os.path.abspath(
716 os.path.join(self._path_to_outdir, self._runtime_deps_path))
721 rel_file_path = os.path.relpath(
722 os.path.abspath(os.path.join(self._path_to_outdir, f)))
745 test = TastTest(args, unknown_args)
747 test = GTestTest(args, unknown_args)
749 test.build_test_command()
751 logging.info(' '.join(test.test_cmd))
753 return test.run_test()
761 '--path-to-outdir must be specified if --deploy-chrome is passed.')
765 '--board',
767 '--cache-dir',
768 os.path.join(CHROMIUM_SRC_PATH, args.cros_cache),
772 '--start',
774 '--copy-on-write',
778 cros_run_test_cmd += ['--device', get_cros_hostname()]
781 '--device', args.device if args.device else LAB_DUT_HOSTNAME
784 cros_run_test_cmd.append('--debug')
786 cros_run_test_cmd.append('--flash')
788 cros_run_test_cmd += ['--public-image']
792 cros_run_test_cmd += ['--results-src', log]
794 '--results-dest-dir',
795 os.path.join(args.logs_dir, 'system_logs')
802 '--deploy-lacros', '--lacros-launcher-script',
806 # Mounting ash-chrome gives it enough disk space to not need stripping
808 cros_run_test_cmd.extend(['--deploy', '--mount'])
810 # Mounting ash-chrome gives it enough disk space to not need stripping
812 cros_run_test_cmd.extend(['--deploy', '--mount'])
815 cros_run_test_cmd.append('--nostrip')
818 '--build-dir',
819 os.path.join(CHROMIUM_SRC_PATH, args.path_to_outdir)
823 '--host-cmd',
824 '--',
835 """Parse hostname from a chromeos-swarming bot id."""
836 for prefix in ['cros-', 'crossk-']:
845 # In chromeos-swarming, we can extract hostname from bot ID, since
860 # Some chromite scripts expect chromite/bin to be on PATH.
861 env['PATH'] = env['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin')
866 # TODO(crbug.com/40567963): Make the GN-dependent deps controllable via cmd
877 parser.add_argument('--verbose', '-v', action='store_true')
879 '--board', type=str, required=True, help='Type of CrOS device.')
881 '--deploy-chrome',
883 help='Will deploy a locally built ash-chrome binary to the device '
884 'before running the host-cmd.')
886 '--deploy-lacros', action='store_true', help='Deploy a lacros-chrome.')
888 '--cros-cache',
891 help='Path to cros cache.')
893 '--path-to-outdir',
896 help='Path to output directory, all of whose contents will be '
899 '--runtime-deps-path',
903 '--vpython-dir',
906 'deploy to the device before the test starts. The location of '
907 'this dir will be added onto PATH in the device. WARNING: The '
911 '--logs-dir',
915 'test into the specified dir.')
918 '--test-launcher-shard-index',
923 '--test-launcher-total-shards',
928 '--flash',
931 'the test.')
933 '--no-flash',
936 help='Will not flash the device before running the test.')
938 '--public-image',
942 '--magic-vm-cache',
943 help='Path to the magic CrOS VM cache dir. See the comment above '
948 '--use-vm',
950 help='Will run the test in the VM instead of a device.')
952 '--device',
954 help='Hostname (or IP) of device to run the test on. This arg is not '
955 'required if --use-vm is set.')
957 '--fetch-cros-hostname',
965 # Host-side test args.
967 'host-cmd',
968 help='Runs a host-side test. Pass the host-side command to run after '
969 '"--". If --use-vm is passed, hostname and port for the device '
973 '--strip-chrome',
975 help='Strips symbols from ash-chrome or lacros-chrome before deploying '
979 'gtest', help='Runs a device-side gtest.')
982 '--test-exe',
985 help='Path to test executable to run inside the device.')
987 # GTest args. Some are passed down to the test binary in the device. Others
990 '--test-launcher-summary-output',
992 help='When set, will pass the same option down to the test and retrieve '
995 '--stop-ui',
997 help='Will stop the UI service in the device before running the test. '
1000 '--trace-dir',
1002 help='When set, will pass down to the test to generate the trace and '
1005 '--env-var',
1009 help='Env var to set on the device for the duration of the test. '
1010 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify '
1013 '--run-test-sudo-helper',
1015 help='When set, will run test_sudo_helper before the test and stop it '
1016 'after test finishes.')
1018 "--no-clean",
1022 help="Do not clean up the deployed files after running the test. "
1023 "Only supported for --remote-cmd tests")
1025 '--set-selinux-label',
1029 ' --set-selinux-label=<filename>=<label>\n'
1031 ' --set-selinux-label=my_test=u:r:cros_foo_label:s0\n'
1034 '--use-deployed-dbus-configs',
1039 # Tast test args.
1040 # pylint: disable=line-too-long
1043 help='Runs a device-side set of Tast tests. For more details, see: '
1048 '--suite-name',
1052 'on what is executed, but is used mainly for test results reporting '
1055 '--test-launcher-summary-output',
1057 help='Generates a simple GTest-style JSON result file for the test run.')
1059 '--attr-expr',
1064 '--strip-chrome',
1066 help='Strips symbols from ash-chrome before deploying to the device.')
1068 '--tast-var',
1074 '--tast-retries',
1079 '--test',
1080 '-t',
1083 help='A Tast test to run in the device (eg: "login.Chrome").')
1085 '--gtest_filter',
1089 'cmd-line API, this will overwrite the value(s) of "--test" above.')
1093 # Re-add N-1 -v/--verbose flags to the args we'll pass to whatever we are
1097 verbose_flags = [a for a in sys.argv if a in ('-v', '--verbose')]
1105 'The test runner is now assuming running in the lab environment, if '
1106 'this is unintentional, please re-invoke the test runner with the '
1107 '"--use-vm" arg if using a VM, otherwise use the "--device=<DUT>" arg '
1111 # we're on a lab bot and are trying to run a test on a lab DUT. See if the
1128 full_vm_cache_path = os.path.join(CHROMIUM_SRC_PATH, args.magic_vm_cache)
1129 if os.path.exists(full_vm_cache_path):
1130 with open(os.path.join(full_vm_cache_path, 'swarming.txt'), 'w') as f:
1131 f.write('non-empty file to make swarming persist this cache')